From 21857784d6d7a9139711cf77f77da925fe9189ee Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 26 Apr 2023 12:53:24 -0600 Subject: [PATCH 001/156] Implement PEP 685 extra normalization in resolver All extras from user input or dependant package metadata are properly normalized for comparison and resolution. This ensures requests for extras from a dependant can always correctly find the normalized extra in the dependency, even if the requested extra name is not normalized. Note that this still relies on the declaration of extra names in the dependency's package metadata to be properly normalized when the package is built, since correct comparison between an extra name's normalized and non-normalized forms requires change to the metadata parsing logic, which is only available in packaging 22.0 and up, which pip does not use at the moment. --- news/11649.bugfix.rst | 5 ++++ .../_internal/resolution/resolvelib/base.py | 2 +- .../resolution/resolvelib/candidates.py | 2 +- .../resolution/resolvelib/factory.py | 20 +++++++++------- .../resolution/resolvelib/requirements.py | 2 +- tests/functional/test_install_extras.py | 24 ++++++++++++++++++- 6 files changed, 42 insertions(+), 13 deletions(-) create mode 100644 news/11649.bugfix.rst diff --git a/news/11649.bugfix.rst b/news/11649.bugfix.rst new file mode 100644 index 000000000..65511711f --- /dev/null +++ b/news/11649.bugfix.rst @@ -0,0 +1,5 @@ +Normalize extras according to :pep:`685` from package metadata in the resolver +for comparison. This ensures extras are correctly compared and merged as long +as the package providing the extra(s) is built with values normalized according +to the standard. Note, however, that this *does not* solve cases where the +package itself contains unnormalized extra values in the metadata. diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index b206692a0..0275385db 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -12,7 +12,7 @@ CandidateLookup = Tuple[Optional["Candidate"], Optional[InstallRequirement]] CandidateVersion = Union[LegacyVersion, Version] -def format_name(project: str, extras: FrozenSet[str]) -> str: +def format_name(project: NormalizedName, extras: FrozenSet[NormalizedName]) -> str: if not extras: return project canonical_extras = sorted(canonicalize_name(e) for e in extras) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 31020e27a..48ef9a16d 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -423,7 +423,7 @@ class ExtrasCandidate(Candidate): def __init__( self, base: BaseCandidate, - extras: FrozenSet[str], + extras: FrozenSet[NormalizedName], ) -> None: self.base = base self.extras = extras diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 0331297b8..6d1ec3163 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -112,7 +112,7 @@ class Factory: self._editable_candidate_cache: Cache[EditableCandidate] = {} self._installed_candidate_cache: Dict[str, AlreadyInstalledCandidate] = {} self._extras_candidate_cache: Dict[ - Tuple[int, FrozenSet[str]], ExtrasCandidate + Tuple[int, FrozenSet[NormalizedName]], ExtrasCandidate ] = {} if not ignore_installed: @@ -138,7 +138,9 @@ class Factory: raise UnsupportedWheel(msg) def _make_extras_candidate( - self, base: BaseCandidate, extras: FrozenSet[str] + self, + base: BaseCandidate, + extras: FrozenSet[NormalizedName], ) -> ExtrasCandidate: cache_key = (id(base), extras) try: @@ -151,7 +153,7 @@ class Factory: def _make_candidate_from_dist( self, dist: BaseDistribution, - extras: FrozenSet[str], + extras: FrozenSet[NormalizedName], template: InstallRequirement, ) -> Candidate: try: @@ -166,7 +168,7 @@ class Factory: def _make_candidate_from_link( self, link: Link, - extras: FrozenSet[str], + extras: FrozenSet[NormalizedName], template: InstallRequirement, name: Optional[NormalizedName], version: Optional[CandidateVersion], @@ -244,12 +246,12 @@ class Factory: assert template.req, "Candidates found on index must be PEP 508" name = canonicalize_name(template.req.name) - extras: FrozenSet[str] = frozenset() + extras: FrozenSet[NormalizedName] = frozenset() for ireq in ireqs: assert ireq.req, "Candidates found on index must be PEP 508" specifier &= ireq.req.specifier hashes &= ireq.hashes(trust_internet=False) - extras |= frozenset(ireq.extras) + extras |= frozenset(canonicalize_name(e) for e in ireq.extras) def _get_installed_candidate() -> Optional[Candidate]: """Get the candidate for the currently-installed version.""" @@ -325,7 +327,7 @@ class Factory: def _iter_explicit_candidates_from_base( self, base_requirements: Iterable[Requirement], - extras: FrozenSet[str], + extras: FrozenSet[NormalizedName], ) -> Iterator[Candidate]: """Produce explicit candidates from the base given an extra-ed package. @@ -392,7 +394,7 @@ class Factory: explicit_candidates.update( self._iter_explicit_candidates_from_base( requirements.get(parsed_requirement.name, ()), - frozenset(parsed_requirement.extras), + frozenset(canonicalize_name(e) for e in parsed_requirement.extras), ), ) @@ -452,7 +454,7 @@ class Factory: self._fail_if_link_is_unsupported_wheel(ireq.link) cand = self._make_candidate_from_link( ireq.link, - extras=frozenset(ireq.extras), + extras=frozenset(canonicalize_name(e) for e in ireq.extras), template=ireq, name=canonicalize_name(ireq.name) if ireq.name else None, version=None, diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index 06addc0dd..7d244c693 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -43,7 +43,7 @@ class SpecifierRequirement(Requirement): def __init__(self, ireq: InstallRequirement) -> None: assert ireq.link is None, "This is a link, not a specifier" self._ireq = ireq - self._extras = frozenset(ireq.extras) + self._extras = frozenset(canonicalize_name(e) for e in ireq.extras) def __str__(self) -> str: return str(self._ireq.req) diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index c6cef00fa..6f2a6bf43 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -4,7 +4,12 @@ from os.path import join import pytest -from tests.lib import PipTestEnvironment, ResolverVariant, TestData +from tests.lib import ( + PipTestEnvironment, + ResolverVariant, + TestData, + create_basic_wheel_for_package, +) @pytest.mark.network @@ -223,3 +228,20 @@ def test_install_extra_merging( if not fails_on_legacy or resolver_variant == "2020-resolver": expected = f"Successfully installed pkga-0.1 simple-{simple_version}" assert expected in result.stdout + + +def test_install_extras(script: PipTestEnvironment) -> None: + create_basic_wheel_for_package(script, "a", "1", depends=["b", "dep[x-y]"]) + create_basic_wheel_for_package(script, "b", "1", depends=["dep[x_y]"]) + create_basic_wheel_for_package(script, "dep", "1", extras={"x-y": ["meh"]}) + create_basic_wheel_for_package(script, "meh", "1") + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "a", + ) + script.assert_installed(a="1", b="1", dep="1", meh="1") From c3160c5423e778b0dc334a677ae865befd222021 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 9 May 2023 15:39:58 +0800 Subject: [PATCH 002/156] Avoid importing things from conftest It is generally discouraged to import from conftest. Things are now moved to tests.lib and imported from there instead. Also did some cleanup to remove the no-longer-needed nullcontext shim. --- tests/conftest.py | 109 +++--------------------- tests/functional/test_completion.py | 3 +- tests/functional/test_download.py | 4 +- tests/functional/test_help.py | 3 +- tests/functional/test_inspect.py | 3 +- tests/functional/test_install.py | 2 +- tests/functional/test_install_config.py | 4 +- tests/functional/test_list.py | 2 +- tests/lib/__init__.py | 48 +++++++++-- tests/lib/compat.py | 23 +---- tests/lib/server.py | 51 ++++++++++- 11 files changed, 114 insertions(+), 138 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 57dd7e68a..5b189443f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,23 +1,21 @@ import compileall +import contextlib import fnmatch -import io import os import re import shutil import subprocess import sys -from contextlib import ExitStack, contextmanager from pathlib import Path from typing import ( - TYPE_CHECKING, AnyStr, Callable, + ContextManager, Dict, Iterable, Iterator, List, Optional, - Union, ) from unittest.mock import patch from zipfile import ZipFile @@ -36,25 +34,20 @@ from installer.destinations import SchemeDictionaryDestination from installer.sources import WheelFile from pip import __file__ as pip_location -from pip._internal.cli.main import main as pip_entry_point from pip._internal.locations import _USE_SYSCONFIG from pip._internal.utils.temp_dir import global_tempdir_manager -from tests.lib import DATA_DIR, SRC_DIR, PipTestEnvironment, TestData -from tests.lib.server import MockServer as _MockServer -from tests.lib.server import make_mock_server, server_running +from tests.lib import ( + DATA_DIR, + SRC_DIR, + CertFactory, + InMemoryPip, + PipTestEnvironment, + ScriptFactory, + TestData, +) +from tests.lib.server import MockServer, make_mock_server from tests.lib.venv import VirtualEnvironment, VirtualEnvironmentType -from .lib.compat import nullcontext - -if TYPE_CHECKING: - from typing import Protocol - - from wsgi import WSGIApplication -else: - # TODO: Protocol was introduced in Python 3.8. Remove this branch when - # dropping support for Python 3.7. - Protocol = object - def pytest_addoption(parser: Parser) -> None: parser.addoption( @@ -325,7 +318,7 @@ def scoped_global_tempdir_manager(request: pytest.FixtureRequest) -> Iterator[No temporary directories in the application. """ if "no_auto_tempdir_manager" in request.keywords: - ctx = nullcontext + ctx: Callable[[], ContextManager[None]] = contextlib.nullcontext else: ctx = global_tempdir_manager @@ -502,16 +495,6 @@ def virtualenv( yield virtualenv_factory(tmpdir.joinpath("workspace", "venv")) -class ScriptFactory(Protocol): - def __call__( - self, - tmpdir: Path, - virtualenv: Optional[VirtualEnvironment] = None, - environ: Optional[Dict[AnyStr, AnyStr]] = None, - ) -> PipTestEnvironment: - ... - - @pytest.fixture(scope="session") def script_factory( virtualenv_factory: Callable[[Path], VirtualEnvironment], @@ -631,26 +614,6 @@ def data(tmpdir: Path) -> TestData: return TestData.copy(tmpdir.joinpath("data")) -class InMemoryPipResult: - def __init__(self, returncode: int, stdout: str) -> None: - self.returncode = returncode - self.stdout = stdout - - -class InMemoryPip: - def pip(self, *args: Union[str, Path]) -> InMemoryPipResult: - orig_stdout = sys.stdout - stdout = io.StringIO() - sys.stdout = stdout - try: - returncode = pip_entry_point([os.fspath(a) for a in args]) - except SystemExit as e: - returncode = e.code or 0 - finally: - sys.stdout = orig_stdout - return InMemoryPipResult(returncode, stdout.getvalue()) - - @pytest.fixture def in_memory_pip() -> InMemoryPip: return InMemoryPip() @@ -662,9 +625,6 @@ def deprecated_python() -> bool: return sys.version_info[:2] in [] -CertFactory = Callable[[], str] - - @pytest.fixture(scope="session") def cert_factory(tmpdir_factory: pytest.TempPathFactory) -> CertFactory: # Delay the import requiring cryptography in order to make it possible @@ -686,49 +646,6 @@ def cert_factory(tmpdir_factory: pytest.TempPathFactory) -> CertFactory: return factory -class MockServer: - def __init__(self, server: _MockServer) -> None: - self._server = server - self._running = False - self.context = ExitStack() - - @property - def port(self) -> int: - return self._server.port - - @property - def host(self) -> str: - return self._server.host - - def set_responses(self, responses: Iterable["WSGIApplication"]) -> None: - assert not self._running, "responses cannot be set on running server" - self._server.mock.side_effect = responses - - def start(self) -> None: - assert not self._running, "running server cannot be started" - self.context.enter_context(server_running(self._server)) - self.context.enter_context(self._set_running()) - - @contextmanager - def _set_running(self) -> Iterator[None]: - self._running = True - try: - yield - finally: - self._running = False - - def stop(self) -> None: - assert self._running, "idle server cannot be stopped" - self.context.close() - - def get_requests(self) -> List[Dict[str, str]]: - """Get environ for each received request.""" - assert not self._running, "cannot get mock from running server" - # Legacy: replace call[0][0] with call.args[0] - # when pip drops support for python3.7 - return [call[0][0] for call in self._server.mock.call_args_list] - - @pytest.fixture def mock_server() -> Iterator[MockServer]: server = make_mock_server() diff --git a/tests/functional/test_completion.py b/tests/functional/test_completion.py index b02cd4fa3..28381c209 100644 --- a/tests/functional/test_completion.py +++ b/tests/functional/test_completion.py @@ -5,8 +5,7 @@ from typing import TYPE_CHECKING, Tuple, Union import pytest -from tests.conftest import ScriptFactory -from tests.lib import PipTestEnvironment, TestData, TestPipResult +from tests.lib import PipTestEnvironment, ScriptFactory, TestData, TestPipResult if TYPE_CHECKING: from typing import Protocol diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 31418ca8c..8c00dc09e 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -14,15 +14,15 @@ import pytest from pip._internal.cli.status_codes import ERROR from pip._internal.utils.urls import path_to_url -from tests.conftest import MockServer, ScriptFactory from tests.lib import ( PipTestEnvironment, + ScriptFactory, TestData, TestPipResult, create_basic_sdist_for_package, create_really_basic_wheel, ) -from tests.lib.server import file_response +from tests.lib.server import MockServer, file_response def fake_wheel(data: TestData, wheel_path: str) -> None: diff --git a/tests/functional/test_help.py b/tests/functional/test_help.py index dba41af5f..69419b8d9 100644 --- a/tests/functional/test_help.py +++ b/tests/functional/test_help.py @@ -5,8 +5,7 @@ import pytest from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.commands import commands_dict, create_command from pip._internal.exceptions import CommandError -from tests.conftest import InMemoryPip -from tests.lib import PipTestEnvironment +from tests.lib import InMemoryPip, PipTestEnvironment def test_run_method_should_return_success_when_finds_command_name() -> None: diff --git a/tests/functional/test_inspect.py b/tests/functional/test_inspect.py index c9f431346..f6690fb1f 100644 --- a/tests/functional/test_inspect.py +++ b/tests/functional/test_inspect.py @@ -2,8 +2,7 @@ import json import pytest -from tests.conftest import ScriptFactory -from tests.lib import PipTestEnvironment, TestData +from tests.lib import PipTestEnvironment, ScriptFactory, TestData @pytest.fixture(scope="session") diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 637128274..c29880e61 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -15,8 +15,8 @@ from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.models.index import PyPI, TestPyPI from pip._internal.utils.misc import rmtree from pip._internal.utils.urls import path_to_url -from tests.conftest import CertFactory from tests.lib import ( + CertFactory, PipTestEnvironment, ResolverVariant, TestData, diff --git a/tests/functional/test_install_config.py b/tests/functional/test_install_config.py index 9f8a80677..ecaf2f705 100644 --- a/tests/functional/test_install_config.py +++ b/tests/functional/test_install_config.py @@ -8,9 +8,9 @@ from typing import Callable, List import pytest -from tests.conftest import CertFactory, MockServer, ScriptFactory -from tests.lib import PipTestEnvironment, TestData +from tests.lib import CertFactory, PipTestEnvironment, ScriptFactory, TestData from tests.lib.server import ( + MockServer, authorization_response, file_response, make_mock_server, diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index bd45f82df..a960f3c4e 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -5,9 +5,9 @@ from pathlib import Path import pytest from pip._internal.models.direct_url import DirectUrl, DirInfo -from tests.conftest import ScriptFactory from tests.lib import ( PipTestEnvironment, + ScriptFactory, TestData, _create_test_package, create_test_package_with_setup, diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 7410072f5..cd0b83d12 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -10,11 +10,12 @@ import textwrap from base64 import urlsafe_b64encode from contextlib import contextmanager from hashlib import sha256 -from io import BytesIO +from io import BytesIO, StringIO from textwrap import dedent from typing import ( TYPE_CHECKING, Any, + AnyStr, Callable, Dict, Iterable, @@ -32,6 +33,7 @@ import pytest from pip._vendor.packaging.utils import canonicalize_name from scripttest import FoundDir, FoundFile, ProcResult, TestFileEnvironment +from pip._internal.cli.main import main as pip_entry_point from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.locations import get_major_minor_version @@ -43,12 +45,12 @@ from tests.lib.venv import VirtualEnvironment from tests.lib.wheel import make_wheel if TYPE_CHECKING: - # Literal was introduced in Python 3.8. - from typing import Literal + from typing import Literal, Protocol ResolverVariant = Literal["resolvelib", "legacy"] -else: - ResolverVariant = str +else: # TODO: Remove this branch when dropping support for Python 3.7. + Protocol = object # Protocol was introduced in Python 3.8. + ResolverVariant = str # Literal was introduced in Python 3.8. DATA_DIR = pathlib.Path(__file__).parent.parent.joinpath("data").resolve() SRC_DIR = pathlib.Path(__file__).resolve().parent.parent.parent @@ -1336,3 +1338,39 @@ def need_svn(fn: _Test) -> _Test: def need_mercurial(fn: _Test) -> _Test: return pytest.mark.mercurial(need_executable("Mercurial", ("hg", "version"))(fn)) + + +class InMemoryPipResult: + def __init__(self, returncode: int, stdout: str) -> None: + self.returncode = returncode + self.stdout = stdout + + +class InMemoryPip: + def pip(self, *args: Union[str, pathlib.Path]) -> InMemoryPipResult: + orig_stdout = sys.stdout + stdout = StringIO() + sys.stdout = stdout + try: + returncode = pip_entry_point([os.fspath(a) for a in args]) + except SystemExit as e: + if isinstance(e.code, int): + returncode = e.code + else: + returncode = int(bool(e.code)) + finally: + sys.stdout = orig_stdout + return InMemoryPipResult(returncode, stdout.getvalue()) + + +class ScriptFactory(Protocol): + def __call__( + self, + tmpdir: pathlib.Path, + virtualenv: Optional[VirtualEnvironment] = None, + environ: Optional[Dict[AnyStr, AnyStr]] = None, + ) -> PipTestEnvironment: + ... + + +CertFactory = Callable[[], str] diff --git a/tests/lib/compat.py b/tests/lib/compat.py index 4d44cbddb..866ac7a77 100644 --- a/tests/lib/compat.py +++ b/tests/lib/compat.py @@ -2,32 +2,13 @@ import contextlib import signal -from typing import Iterable, Iterator - - -@contextlib.contextmanager -def nullcontext() -> Iterator[None]: - """ - Context manager that does no additional processing. - - Used as a stand-in for a normal context manager, when a particular block of - code is only sometimes used with a normal context manager: - - cm = optional_cm if condition else nullcontext() - with cm: - # Perform operation, using optional_cm if condition is True - - TODO: Replace with contextlib.nullcontext after dropping Python 3.6 - support. - """ - yield - +from typing import Callable, ContextManager, Iterable, Iterator # Applies on Windows. if not hasattr(signal, "pthread_sigmask"): # We're not relying on this behavior anywhere currently, it's just best # practice. - blocked_signals = nullcontext + blocked_signals: Callable[[], ContextManager[None]] = contextlib.nullcontext else: @contextlib.contextmanager diff --git a/tests/lib/server.py b/tests/lib/server.py index 4cc18452c..1048a173d 100644 --- a/tests/lib/server.py +++ b/tests/lib/server.py @@ -2,9 +2,9 @@ import pathlib import ssl import threading from base64 import b64encode -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager from textwrap import dedent -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator, List from unittest.mock import Mock from werkzeug.serving import BaseWSGIServer, WSGIRequestHandler @@ -18,7 +18,7 @@ if TYPE_CHECKING: Body = Iterable[bytes] -class MockServer(BaseWSGIServer): +class _MockServer(BaseWSGIServer): mock: Mock = Mock() @@ -64,7 +64,7 @@ def _mock_wsgi_adapter( return adapter -def make_mock_server(**kwargs: Any) -> MockServer: +def make_mock_server(**kwargs: Any) -> _MockServer: """Creates a mock HTTP(S) server listening on a random port on localhost. The `mock` property of the returned server provides and records all WSGI @@ -189,3 +189,46 @@ def authorization_response(path: pathlib.Path) -> "WSGIApplication": return [path.read_bytes()] return responder + + +class MockServer: + def __init__(self, server: _MockServer) -> None: + self._server = server + self._running = False + self.context = ExitStack() + + @property + def port(self) -> int: + return self._server.port + + @property + def host(self) -> str: + return self._server.host + + def set_responses(self, responses: Iterable["WSGIApplication"]) -> None: + assert not self._running, "responses cannot be set on running server" + self._server.mock.side_effect = responses + + def start(self) -> None: + assert not self._running, "running server cannot be started" + self.context.enter_context(server_running(self._server)) + self.context.enter_context(self._set_running()) + + @contextmanager + def _set_running(self) -> Iterator[None]: + self._running = True + try: + yield + finally: + self._running = False + + def stop(self) -> None: + assert self._running, "idle server cannot be stopped" + self.context.close() + + def get_requests(self) -> List[Dict[str, str]]: + """Get environ for each received request.""" + assert not self._running, "cannot get mock from running server" + # Legacy: replace call[0][0] with call.args[0] + # when pip drops support for python3.7 + return [call[0][0] for call in self._server.mock.call_args_list] From b9066d4b00a2fa2cc6529ecb0b5920465e0fb812 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 11 May 2023 13:00:47 +0800 Subject: [PATCH 003/156] Add test cases for normalized weird extra --- tests/functional/test_install_extras.py | 33 ++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index 6f2a6bf43..21da9d50e 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -155,25 +155,50 @@ def test_install_fails_if_extra_at_end( assert "Extras after version" in result.stderr -def test_install_special_extra(script: PipTestEnvironment) -> None: +@pytest.mark.parametrize( + "specified_extra, requested_extra", + [ + ("Hop_hOp-hoP", "Hop_hOp-hoP"), + pytest.param( + "Hop_hOp-hoP", + "hop-hop-hop", + marks=pytest.mark.xfail( + reason=( + "matching a normalized extra request against an" + "unnormalized extra in metadata requires PEP 685 support " + "in packaging (see pypa/pip#11445)." + ), + ), + ), + ("hop-hop-hop", "Hop_hOp-hoP"), + ], +) +def test_install_special_extra( + script: PipTestEnvironment, + specified_extra: str, + requested_extra: str, +) -> None: # Check that uppercase letters and '-' are dealt with # make a dummy project pkga_path = script.scratch_path / "pkga" pkga_path.mkdir() pkga_path.joinpath("setup.py").write_text( textwrap.dedent( - """ + f""" from setuptools import setup setup(name='pkga', version='0.1', - extras_require={'Hop_hOp-hoP': ['missing_pkg']}, + extras_require={{'{specified_extra}': ['missing_pkg']}}, ) """ ) ) result = script.pip( - "install", "--no-index", f"{pkga_path}[Hop_hOp-hoP]", expect_error=True + "install", + "--no-index", + f"{pkga_path}[{requested_extra}]", + expect_error=True, ) assert ( "Could not find a version that satisfies the requirement missing_pkg" From d64190c5fbbf38cf40215ef7122f1b8c6847afc9 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 11 May 2023 14:32:41 +0800 Subject: [PATCH 004/156] Try to find dependencies from unnormalized extras When an unnormalized extra is requested, try to look up dependencies with both its raw and normalized forms, to maintain compatibility when an extra is both specified and requested in a non-standard form. --- .../resolution/resolvelib/candidates.py | 62 ++++++++++++++----- .../resolution/resolvelib/factory.py | 18 +++--- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 48ef9a16d..b737bffc9 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -423,10 +423,17 @@ class ExtrasCandidate(Candidate): def __init__( self, base: BaseCandidate, - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], ) -> None: self.base = base - self.extras = extras + self.extras = frozenset(canonicalize_name(e) for e in extras) + # If any extras are requested in their non-normalized forms, keep track + # of their raw values. This is needed when we look up dependencies + # since PEP 685 has not been implemented for marker-matching, and using + # the non-normalized extra for lookup ensures the user can select a + # non-normalized extra in a package with its non-normalized form. + # TODO: Remove this when packaging is upgraded to support PEP 685. + self._unnormalized_extras = extras.difference(self.extras) def __str__(self) -> str: name, rest = str(self.base).split(" ", 1) @@ -477,6 +484,44 @@ class ExtrasCandidate(Candidate): def source_link(self) -> Optional[Link]: return self.base.source_link + def _warn_invalid_extras( + self, + requested: FrozenSet[str], + provided: FrozenSet[str], + ) -> None: + """Emit warnings for invalid extras being requested. + + This emits a warning for each requested extra that is not in the + candidate's ``Provides-Extra`` list. + """ + invalid_extras_to_warn = requested.difference( + provided, + # If an extra is requested in an unnormalized form, skip warning + # about the normalized form being missing. + (canonicalize_name(e) for e in self._unnormalized_extras), + ) + if not invalid_extras_to_warn: + return + for extra in sorted(invalid_extras_to_warn): + logger.warning( + "%s %s does not provide the extra '%s'", + self.base.name, + self.version, + extra, + ) + + def _calculate_valid_requested_extras(self) -> FrozenSet[str]: + """Get a list of valid extras requested by this candidate. + + The user (or upstream dependant) may have specified extras that the + candidate doesn't support. Any unsupported extras are dropped, and each + cause a warning to be logged here. + """ + requested_extras = self.extras.union(self._unnormalized_extras) + provided_extras = frozenset(self.base.dist.iter_provided_extras()) + self._warn_invalid_extras(requested_extras, provided_extras) + return requested_extras.intersection(provided_extras) + def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]: factory = self.base._factory @@ -486,18 +531,7 @@ class ExtrasCandidate(Candidate): if not with_requires: return - # The user may have specified extras that the candidate doesn't - # support. We ignore any unsupported extras here. - valid_extras = self.extras.intersection(self.base.dist.iter_provided_extras()) - invalid_extras = self.extras.difference(self.base.dist.iter_provided_extras()) - for extra in sorted(invalid_extras): - logger.warning( - "%s %s does not provide the extra '%s'", - self.base.name, - self.version, - extra, - ) - + valid_extras = self._calculate_valid_requested_extras() for r in self.base.dist.iter_dependencies(valid_extras): requirement = factory.make_requirement_from_spec( str(r), self.base._ireq, valid_extras diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 6d1ec3163..ff916236c 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -140,9 +140,9 @@ class Factory: def _make_extras_candidate( self, base: BaseCandidate, - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], ) -> ExtrasCandidate: - cache_key = (id(base), extras) + cache_key = (id(base), frozenset(canonicalize_name(e) for e in extras)) try: candidate = self._extras_candidate_cache[cache_key] except KeyError: @@ -153,7 +153,7 @@ class Factory: def _make_candidate_from_dist( self, dist: BaseDistribution, - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], template: InstallRequirement, ) -> Candidate: try: @@ -168,7 +168,7 @@ class Factory: def _make_candidate_from_link( self, link: Link, - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], template: InstallRequirement, name: Optional[NormalizedName], version: Optional[CandidateVersion], @@ -246,12 +246,12 @@ class Factory: assert template.req, "Candidates found on index must be PEP 508" name = canonicalize_name(template.req.name) - extras: FrozenSet[NormalizedName] = frozenset() + extras: FrozenSet[str] = frozenset() for ireq in ireqs: assert ireq.req, "Candidates found on index must be PEP 508" specifier &= ireq.req.specifier hashes &= ireq.hashes(trust_internet=False) - extras |= frozenset(canonicalize_name(e) for e in ireq.extras) + extras |= frozenset(ireq.extras) def _get_installed_candidate() -> Optional[Candidate]: """Get the candidate for the currently-installed version.""" @@ -327,7 +327,7 @@ class Factory: def _iter_explicit_candidates_from_base( self, base_requirements: Iterable[Requirement], - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], ) -> Iterator[Candidate]: """Produce explicit candidates from the base given an extra-ed package. @@ -394,7 +394,7 @@ class Factory: explicit_candidates.update( self._iter_explicit_candidates_from_base( requirements.get(parsed_requirement.name, ()), - frozenset(canonicalize_name(e) for e in parsed_requirement.extras), + frozenset(parsed_requirement.extras), ), ) @@ -454,7 +454,7 @@ class Factory: self._fail_if_link_is_unsupported_wheel(ireq.link) cand = self._make_candidate_from_link( ireq.link, - extras=frozenset(canonicalize_name(e) for e in ireq.extras), + extras=frozenset(ireq.extras), template=ireq, name=canonicalize_name(ireq.name) if ireq.name else None, version=None, From 4aa6d88ddcccde3e0f189b447f0c8886ceebe008 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 11 May 2023 15:12:30 +0800 Subject: [PATCH 005/156] Remove extra normalization from format_name util Since this function now always take normalized names, additional normalization is no longer needed. --- src/pip/_internal/resolution/resolvelib/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index 0275385db..9c0ef5ca7 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -1,7 +1,7 @@ from typing import FrozenSet, Iterable, Optional, Tuple, Union from pip._vendor.packaging.specifiers import SpecifierSet -from pip._vendor.packaging.utils import NormalizedName, canonicalize_name +from pip._vendor.packaging.utils import NormalizedName from pip._vendor.packaging.version import LegacyVersion, Version from pip._internal.models.link import Link, links_equivalent @@ -15,8 +15,8 @@ CandidateVersion = Union[LegacyVersion, Version] def format_name(project: NormalizedName, extras: FrozenSet[NormalizedName]) -> str: if not extras: return project - canonical_extras = sorted(canonicalize_name(e) for e in extras) - return "{}[{}]".format(project, ",".join(canonical_extras)) + extras_expr = ",".join(sorted(extras)) + return f"{project}[{extras_expr}]" class Constraint: From 5bebe850ea1db6ff165e4b95bafb1ee44e4a69e8 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 20 Jun 2023 17:13:18 +0200 Subject: [PATCH 006/156] take non-extra requirements into account for extra installs --- src/pip/_internal/resolution/resolvelib/factory.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 0331297b8..c117a30c8 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -385,8 +385,8 @@ class Factory: if ireq is not None: ireqs.append(ireq) - # If the current identifier contains extras, add explicit candidates - # from entries from extra-less identifier. + # If the current identifier contains extras, add requires and explicit + # candidates from entries from extra-less identifier. with contextlib.suppress(InvalidRequirement): parsed_requirement = get_requirement(identifier) explicit_candidates.update( @@ -395,6 +395,10 @@ class Factory: frozenset(parsed_requirement.extras), ), ) + for req in requirements.get(parsed_requirement.name, []): + _, ireq = req.get_candidate_lookup() + if ireq is not None: + ireqs.append(ireq) # Add explicit candidates from constraints. We only do this if there are # known ireqs, which represent requirements not already explicit. If From 937d8f0b61dbf41f23db9ba62586a6bf6d45c828 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 21 Jun 2023 17:34:30 +0200 Subject: [PATCH 007/156] partial improvement --- src/pip/_internal/req/constructors.py | 32 +++++++++++- .../resolution/resolvelib/candidates.py | 9 ++-- .../resolution/resolvelib/factory.py | 51 ++++++++++++++----- .../resolution/resolvelib/provider.py | 2 + .../resolution/resolvelib/requirements.py | 28 ++++++++-- .../resolution_resolvelib/test_requirement.py | 22 ++++---- 6 files changed, 109 insertions(+), 35 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index c5ca2d85d..f04a4cbbd 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -15,7 +15,7 @@ from typing import Dict, List, Optional, Set, Tuple, Union from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import InvalidRequirement, Requirement -from pip._vendor.packaging.specifiers import Specifier +from pip._vendor.packaging.specifiers import Specifier, SpecifierSet from pip._internal.exceptions import InstallationError from pip._internal.models.index import PyPI, TestPyPI @@ -504,3 +504,33 @@ def install_req_from_link_and_ireq( config_settings=ireq.config_settings, user_supplied=ireq.user_supplied, ) + + +def install_req_without( + ireq: InstallRequirement, + *, + without_extras: bool = False, + without_specifier: bool = False, +) -> InstallRequirement: + # TODO: clean up hack + req = Requirement(str(ireq.req)) + if without_extras: + req.extras = {} + if without_specifier: + req.specifier = SpecifierSet(prereleases=req.specifier.prereleases) + return InstallRequirement( + req=req, + comes_from=ireq.comes_from, + editable=ireq.editable, + link=ireq.link, + markers=ireq.markers, + use_pep517=ireq.use_pep517, + isolated=ireq.isolated, + global_options=ireq.global_options, + hash_options=ireq.hash_options, + constraint=ireq.constraint, + extras=ireq.extras if not without_extras else [], + config_settings=ireq.config_settings, + user_supplied=ireq.user_supplied, + permit_editable_wheels=ireq.permit_editable_wheels, + ) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index de04e1d73..5bac3d6df 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -237,10 +237,11 @@ class _InstallRequirementBackedCandidate(Candidate): self._check_metadata_consistency(dist) return dist + # TODO: add Explicit dependency on self to extra reqs can benefit from it? def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]: requires = self.dist.iter_dependencies() if with_requires else () for r in requires: - yield self._factory.make_requirement_from_spec(str(r), self._ireq) + yield from self._factory.make_requirements_from_spec(str(r), self._ireq) yield self._factory.make_requires_python_requirement(self.dist.requires_python) def get_install_requirement(self) -> Optional[InstallRequirement]: @@ -392,7 +393,7 @@ class AlreadyInstalledCandidate(Candidate): if not with_requires: return for r in self.dist.iter_dependencies(): - yield self._factory.make_requirement_from_spec(str(r), self._ireq) + yield from self._factory.make_requirements_from_spec(str(r), self._ireq) def get_install_requirement(self) -> Optional[InstallRequirement]: return None @@ -502,11 +503,9 @@ class ExtrasCandidate(Candidate): ) for r in self.base.dist.iter_dependencies(valid_extras): - requirement = factory.make_requirement_from_spec( + yield from factory.make_requirements_from_spec( str(r), self.base._ireq, valid_extras ) - if requirement: - yield requirement def get_install_requirement(self) -> Optional[InstallRequirement]: # We don't return anything here, because we always diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index c117a30c8..4c088209b 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -441,18 +441,35 @@ class Factory: and all(req.is_satisfied_by(c) for req in requirements[identifier]) ) - def _make_requirement_from_install_req( + def _make_requirements_from_install_req( self, ireq: InstallRequirement, requested_extras: Iterable[str] - ) -> Optional[Requirement]: + ) -> list[Requirement]: + # TODO: docstring + """ + Returns requirement objects associated with the given InstallRequirement. In + most cases this will be a single object but the following special cases exist: + - the InstallRequirement has markers that do not apply -> result is empty + - the InstallRequirement has both a constraint and extras -> result is split + in two requirement objects: one with the constraint and one with the + extra. This allows centralized constraint handling for the base, + resulting in fewer candidate rejections. + """ + # TODO: implement -> split in base req with constraint and extra req without if not ireq.match_markers(requested_extras): logger.info( "Ignoring %s: markers '%s' don't match your environment", ireq.name, ireq.markers, ) - return None + return [] if not ireq.link: - return SpecifierRequirement(ireq) + if ireq.extras and ireq.req.specifier: + return [ + SpecifierRequirement(ireq, drop_extras=True), + SpecifierRequirement(ireq, drop_specifier=True), + ] + else: + return [SpecifierRequirement(ireq)] self._fail_if_link_is_unsupported_wheel(ireq.link) cand = self._make_candidate_from_link( ireq.link, @@ -470,8 +487,9 @@ class Factory: # ResolutionImpossible eventually. if not ireq.name: raise self._build_failures[ireq.link] - return UnsatisfiableRequirement(canonicalize_name(ireq.name)) - return self.make_requirement_from_candidate(cand) + return [UnsatisfiableRequirement(canonicalize_name(ireq.name))] + # TODO: here too + return [self.make_requirement_from_candidate(cand)] def collect_root_requirements( self, root_ireqs: List[InstallRequirement] @@ -492,15 +510,17 @@ class Factory: else: collected.constraints[name] = Constraint.from_ireq(ireq) else: - req = self._make_requirement_from_install_req( + reqs = self._make_requirements_from_install_req( ireq, requested_extras=(), ) - if req is None: + if not reqs: continue - if ireq.user_supplied and req.name not in collected.user_requested: - collected.user_requested[req.name] = i - collected.requirements.append(req) + + # TODO: clean up reqs[0]? + if ireq.user_supplied and reqs[0].name not in collected.user_requested: + collected.user_requested[reqs[0].name] = i + collected.requirements.extend(reqs) return collected def make_requirement_from_candidate( @@ -508,14 +528,17 @@ class Factory: ) -> ExplicitRequirement: return ExplicitRequirement(candidate) - def make_requirement_from_spec( + def make_requirements_from_spec( self, specifier: str, comes_from: Optional[InstallRequirement], requested_extras: Iterable[str] = (), - ) -> Optional[Requirement]: + ) -> list[Requirement]: + # TODO: docstring + """ + """ ireq = self._make_install_req_from_spec(specifier, comes_from) - return self._make_requirement_from_install_req(ireq, requested_extras) + return self._make_requirements_from_install_req(ireq, requested_extras) def make_requires_python_requirement( self, diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 315fb9c89..121e48d07 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -184,6 +184,8 @@ class PipProvider(_ProviderBase): # the backtracking backtrack_cause = self.is_backtrack_cause(identifier, backtrack_causes) + # TODO: finally prefer base over extra for the same package + return ( not requires_python, not direct, diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index 06addc0dd..fe9ae6ba6 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -2,6 +2,7 @@ from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import NormalizedName, canonicalize_name from pip._internal.req.req_install import InstallRequirement +from pip._internal.req.constructors import install_req_without from .base import Candidate, CandidateLookup, Requirement, format_name @@ -39,14 +40,27 @@ class ExplicitRequirement(Requirement): return candidate == self.candidate +# TODO: add some comments class SpecifierRequirement(Requirement): - def __init__(self, ireq: InstallRequirement) -> None: + # TODO: document additional options + def __init__( + self, + ireq: InstallRequirement, + *, + drop_extras: bool = False, + drop_specifier: bool = False, + ) -> None: assert ireq.link is None, "This is a link, not a specifier" - self._ireq = ireq - self._extras = frozenset(ireq.extras) + self._drop_extras: bool = drop_extras + self._original_extras = frozenset(ireq.extras) + # TODO: name + self._original_req = ireq.req + self._ireq = install_req_without( + ireq, without_extras=self._drop_extras, without_specifier=drop_specifier + ) def __str__(self) -> str: - return str(self._ireq.req) + return str(self._original_req) def __repr__(self) -> str: return "{class_name}({requirement!r})".format( @@ -59,9 +73,13 @@ class SpecifierRequirement(Requirement): assert self._ireq.req, "Specifier-backed ireq is always PEP 508" return canonicalize_name(self._ireq.req.name) + # TODO: make sure this can still be identified for error reporting purposes @property def name(self) -> str: - return format_name(self.project_name, self._extras) + return format_name( + self.project_name, + self._original_extras if not self._drop_extras else frozenset(), + ) def format_for_error(self) -> str: # Convert comma-separated specifiers into "A, B, ..., F and G" diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index 6864e70ea..ce48ab16c 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -61,9 +61,9 @@ def test_new_resolver_requirement_has_name( ) -> None: """All requirements should have a name""" for spec, name, _ in test_cases: - req = factory.make_requirement_from_spec(spec, comes_from=None) - assert req is not None - assert req.name == name + reqs = factory.make_requirements_from_spec(spec, comes_from=None) + assert len(reqs) == 1 + assert reqs[0].name == name def test_new_resolver_correct_number_of_matches( @@ -71,8 +71,9 @@ def test_new_resolver_correct_number_of_matches( ) -> None: """Requirements should return the correct number of candidates""" for spec, _, match_count in test_cases: - req = factory.make_requirement_from_spec(spec, comes_from=None) - assert req is not None + reqs = factory.make_requirements_from_spec(spec, comes_from=None) + assert len(reqs) == 1 + req = reqs[0] matches = factory.find_candidates( req.name, {req.name: [req]}, @@ -88,8 +89,9 @@ def test_new_resolver_candidates_match_requirement( ) -> None: """Candidates returned from find_candidates should satisfy the requirement""" for spec, _, _ in test_cases: - req = factory.make_requirement_from_spec(spec, comes_from=None) - assert req is not None + reqs = factory.make_requirements_from_spec(spec, comes_from=None) + assert len(reqs) == 1 + req = reqs[0] candidates = factory.find_candidates( req.name, {req.name: [req]}, @@ -104,8 +106,8 @@ def test_new_resolver_candidates_match_requirement( def test_new_resolver_full_resolve(factory: Factory, provider: PipProvider) -> None: """A very basic full resolve""" - req = factory.make_requirement_from_spec("simplewheel", comes_from=None) - assert req is not None + reqs = factory.make_requirements_from_spec("simplewheel", comes_from=None) + assert len(reqs) == 1 r: Resolver[Requirement, Candidate, str] = Resolver(provider, BaseReporter()) - result = r.resolve([req]) + result = r.resolve([reqs[0]]) assert set(result.mapping.keys()) == {"simplewheel"} From 5f8f40eb1d0610e530d5e035ba8c7f99d9af9df1 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 22 Jun 2023 11:08:33 +0200 Subject: [PATCH 008/156] refinements --- src/pip/_internal/req/constructors.py | 3 +- .../resolution/resolvelib/candidates.py | 15 +++++++- .../resolution/resolvelib/factory.py | 38 +++++++++++-------- .../resolution/resolvelib/requirements.py | 18 ++++----- 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index f04a4cbbd..908876c4c 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -520,7 +520,8 @@ def install_req_without( req.specifier = SpecifierSet(prereleases=req.specifier.prereleases) return InstallRequirement( req=req, - comes_from=ireq.comes_from, + # TODO: document this!!!! + comes_from=ireq, editable=ireq.editable, link=ireq.link, markers=ireq.markers, diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 5bac3d6df..238834841 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -237,7 +237,6 @@ class _InstallRequirementBackedCandidate(Candidate): self._check_metadata_consistency(dist) return dist - # TODO: add Explicit dependency on self to extra reqs can benefit from it? def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]: requires = self.dist.iter_dependencies() if with_requires else () for r in requires: @@ -428,9 +427,19 @@ class ExtrasCandidate(Candidate): self, base: BaseCandidate, extras: FrozenSet[str], + ireq: Optional[InstallRequirement] = None, ) -> None: + """ + :param ireq: the InstallRequirement that led to this candidate, if it + differs from the base's InstallRequirement. This will often be the + case in the sense that this candidate's requirement has the extras + while the base's does not. Unlike the InstallRequirement backed + candidates, this requirement is used solely for reporting purposes, + it does not do any leg work. + """ self.base = base self.extras = extras + self._ireq = ireq def __str__(self) -> str: name, rest = str(self.base).split(" ", 1) @@ -504,7 +513,9 @@ class ExtrasCandidate(Candidate): for r in self.base.dist.iter_dependencies(valid_extras): yield from factory.make_requirements_from_spec( - str(r), self.base._ireq, valid_extras + str(r), + self._ireq if self._ireq is not None else self.base._ireq, + valid_extras, ) def get_install_requirement(self) -> Optional[InstallRequirement]: diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 4c088209b..45b813387 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -138,13 +138,16 @@ class Factory: raise UnsupportedWheel(msg) def _make_extras_candidate( - self, base: BaseCandidate, extras: FrozenSet[str] + self, + base: BaseCandidate, + extras: FrozenSet[str], + ireq: Optional[InstallRequirement] = None, ) -> ExtrasCandidate: cache_key = (id(base), extras) try: candidate = self._extras_candidate_cache[cache_key] except KeyError: - candidate = ExtrasCandidate(base, extras) + candidate = ExtrasCandidate(base, extras, ireq=ireq) self._extras_candidate_cache[cache_key] = candidate return candidate @@ -161,7 +164,7 @@ class Factory: self._installed_candidate_cache[dist.canonical_name] = base if not extras: return base - return self._make_extras_candidate(base, extras) + return self._make_extras_candidate(base, extras, ireq=template) def _make_candidate_from_link( self, @@ -223,7 +226,7 @@ class Factory: if not extras: return base - return self._make_extras_candidate(base, extras) + return self._make_extras_candidate(base, extras, ireq=template) def _iter_found_candidates( self, @@ -389,16 +392,17 @@ class Factory: # candidates from entries from extra-less identifier. with contextlib.suppress(InvalidRequirement): parsed_requirement = get_requirement(identifier) - explicit_candidates.update( - self._iter_explicit_candidates_from_base( - requirements.get(parsed_requirement.name, ()), - frozenset(parsed_requirement.extras), - ), - ) - for req in requirements.get(parsed_requirement.name, []): - _, ireq = req.get_candidate_lookup() - if ireq is not None: - ireqs.append(ireq) + if parsed_requirement.name != identifier: + explicit_candidates.update( + self._iter_explicit_candidates_from_base( + requirements.get(parsed_requirement.name, ()), + frozenset(parsed_requirement.extras), + ), + ) + for req in requirements.get(parsed_requirement.name, []): + _, ireq = req.get_candidate_lookup() + if ireq is not None: + ireqs.append(ireq) # Add explicit candidates from constraints. We only do this if there are # known ireqs, which represent requirements not already explicit. If @@ -444,7 +448,6 @@ class Factory: def _make_requirements_from_install_req( self, ireq: InstallRequirement, requested_extras: Iterable[str] ) -> list[Requirement]: - # TODO: docstring """ Returns requirement objects associated with the given InstallRequirement. In most cases this will be a single object but the following special cases exist: @@ -454,7 +457,6 @@ class Factory: extra. This allows centralized constraint handling for the base, resulting in fewer candidate rejections. """ - # TODO: implement -> split in base req with constraint and extra req without if not ireq.match_markers(requested_extras): logger.info( "Ignoring %s: markers '%s' don't match your environment", @@ -466,6 +468,10 @@ class Factory: if ireq.extras and ireq.req.specifier: return [ SpecifierRequirement(ireq, drop_extras=True), + # TODO: put this all the way at the back to have even fewer candidates? + # TODO: probably best to keep specifier as it makes the report + # slightly more readable -> should also update SpecReq constructor + # and req.constructors.install_req_without SpecifierRequirement(ireq, drop_specifier=True), ] else: diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index fe9ae6ba6..180158128 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -40,7 +40,6 @@ class ExplicitRequirement(Requirement): return candidate == self.candidate -# TODO: add some comments class SpecifierRequirement(Requirement): # TODO: document additional options def __init__( @@ -52,15 +51,17 @@ class SpecifierRequirement(Requirement): ) -> None: assert ireq.link is None, "This is a link, not a specifier" self._drop_extras: bool = drop_extras - self._original_extras = frozenset(ireq.extras) - # TODO: name - self._original_req = ireq.req - self._ireq = install_req_without( - ireq, without_extras=self._drop_extras, without_specifier=drop_specifier + self._extras = frozenset(ireq.extras if not drop_extras else ()) + self._ireq = ( + ireq + if not drop_extras and not drop_specifier + else install_req_without( + ireq, without_extras=self._drop_extras, without_specifier=drop_specifier + ) ) def __str__(self) -> str: - return str(self._original_req) + return str(self._ireq) def __repr__(self) -> str: return "{class_name}({requirement!r})".format( @@ -73,12 +74,11 @@ class SpecifierRequirement(Requirement): assert self._ireq.req, "Specifier-backed ireq is always PEP 508" return canonicalize_name(self._ireq.req.name) - # TODO: make sure this can still be identified for error reporting purposes @property def name(self) -> str: return format_name( self.project_name, - self._original_extras if not self._drop_extras else frozenset(), + self._extras, ) def format_for_error(self) -> str: From d09431feb5049ec5e7a9b4ecb5d338a38a14ffc4 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 22 Jun 2023 14:42:05 +0200 Subject: [PATCH 009/156] fixes --- src/pip/_internal/req/constructors.py | 20 +++++++++++++++++-- .../resolution/resolvelib/resolver.py | 15 +++++++++++++- tests/functional/test_install.py | 6 +++--- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 908876c4c..9bf1c9844 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -8,10 +8,11 @@ These are meant to be used elsewhere within pip to create instances of InstallRequirement. """ +import copy import logging import os import re -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Collection, Dict, List, Optional, Set, Tuple, Union from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import InvalidRequirement, Requirement @@ -512,7 +513,6 @@ def install_req_without( without_extras: bool = False, without_specifier: bool = False, ) -> InstallRequirement: - # TODO: clean up hack req = Requirement(str(ireq.req)) if without_extras: req.extras = {} @@ -535,3 +535,19 @@ def install_req_without( user_supplied=ireq.user_supplied, permit_editable_wheels=ireq.permit_editable_wheels, ) + + +def install_req_extend_extras( + ireq: InstallRequirement, + extras: Collection[str], +) -> InstallRequirement: + """ + Returns a copy of an installation requirement with some additional extras. + Makes a shallow copy of the ireq object. + """ + result = copy.copy(ireq) + req = Requirement(str(ireq.req)) + req.extras.update(extras) + result.req = req + result.extras = {*ireq.extras, *extras} + return result diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 47bbfecce..c5de0e822 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -1,3 +1,4 @@ +import contextlib import functools import logging import os @@ -11,6 +12,7 @@ from pip._vendor.resolvelib.structs import DirectedGraph from pip._internal.cache import WheelCache from pip._internal.index.package_finder import PackageFinder from pip._internal.operations.prepare import RequirementPreparer +from pip._internal.req.constructors import install_req_extend_extras from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_set import RequirementSet from pip._internal.resolution.base import BaseResolver, InstallRequirementProvider @@ -19,6 +21,7 @@ from pip._internal.resolution.resolvelib.reporter import ( PipDebuggingReporter, PipReporter, ) +from pip._internal.utils.packaging import get_requirement from .base import Candidate, Requirement from .factory import Factory @@ -101,9 +104,19 @@ class Resolver(BaseResolver): raise error from e req_set = RequirementSet(check_supported_wheels=check_supported_wheels) - for candidate in result.mapping.values(): + # sort to ensure base candidates come before candidates with extras + for candidate in sorted(result.mapping.values(), key=lambda c: c.name): ireq = candidate.get_install_requirement() if ireq is None: + if candidate.name != candidate.project_name: + # extend existing req's extras + with contextlib.suppress(KeyError): + req = req_set.get_requirement(candidate.project_name) + req_set.add_named_requirement( + install_req_extend_extras( + req, get_requirement(candidate.name).extras + ) + ) continue # Check if there is already an installation under the same name, diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 8559d9368..f5ac31a8e 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -2465,6 +2465,6 @@ def test_install_pip_prints_req_chain_pypi(script: PipTestEnvironment) -> None: ) assert ( - f"Collecting python-openid " - f"(from Paste[openid]==1.7.5.1->-r {req_path} (line 1))" in result.stdout - ) + "Collecting python-openid " + f"(from Paste[openid]->Paste[openid]==1.7.5.1->-r {req_path} (line 1))" + ) in result.stdout From 49027d7de3c9441b612c65ac68ec39e893a8385f Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 22 Jun 2023 14:59:43 +0200 Subject: [PATCH 010/156] cleanup --- src/pip/_internal/req/constructors.py | 20 +++++------ .../resolution/resolvelib/factory.py | 34 ++++++++++++------- .../resolution/resolvelib/requirements.py | 18 ++++------ 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 9bf1c9844..8b1438afe 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -507,20 +507,16 @@ def install_req_from_link_and_ireq( ) -def install_req_without( - ireq: InstallRequirement, - *, - without_extras: bool = False, - without_specifier: bool = False, -) -> InstallRequirement: +def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement: + """ + Creates a new InstallationRequirement using the given template but without + any extras. Sets the original requirement as the new one's parent + (comes_from). + """ req = Requirement(str(ireq.req)) - if without_extras: - req.extras = {} - if without_specifier: - req.specifier = SpecifierSet(prereleases=req.specifier.prereleases) + req.extras = {} return InstallRequirement( req=req, - # TODO: document this!!!! comes_from=ireq, editable=ireq.editable, link=ireq.link, @@ -530,7 +526,7 @@ def install_req_without( global_options=ireq.global_options, hash_options=ireq.hash_options, constraint=ireq.constraint, - extras=ireq.extras if not without_extras else [], + extras=[], config_settings=ireq.config_settings, user_supplied=ireq.user_supplied, permit_editable_wheels=ireq.permit_editable_wheels, diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 45b813387..0c2c6ab79 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -468,18 +468,18 @@ class Factory: if ireq.extras and ireq.req.specifier: return [ SpecifierRequirement(ireq, drop_extras=True), - # TODO: put this all the way at the back to have even fewer candidates? - # TODO: probably best to keep specifier as it makes the report - # slightly more readable -> should also update SpecReq constructor - # and req.constructors.install_req_without - SpecifierRequirement(ireq, drop_specifier=True), + # TODO: put this all the way at the back to have even fewer + # candidates? + SpecifierRequirement(ireq), ] else: return [SpecifierRequirement(ireq)] self._fail_if_link_is_unsupported_wheel(ireq.link) cand = self._make_candidate_from_link( ireq.link, - extras=frozenset(ireq.extras), + # make just the base candidate so the corresponding requirement can be split + # in case of extras (see docstring) + extras=frozenset(), template=ireq, name=canonicalize_name(ireq.name) if ireq.name else None, version=None, @@ -494,8 +494,12 @@ class Factory: if not ireq.name: raise self._build_failures[ireq.link] return [UnsatisfiableRequirement(canonicalize_name(ireq.name))] - # TODO: here too - return [self.make_requirement_from_candidate(cand)] + return [ + self.make_requirement_from_candidate(cand), + self.make_requirement_from_candidate( + self._make_extras_candidate(cand, frozenset(ireq.extras), ireq) + ), + ] def collect_root_requirements( self, root_ireqs: List[InstallRequirement] @@ -523,9 +527,9 @@ class Factory: if not reqs: continue - # TODO: clean up reqs[0]? - if ireq.user_supplied and reqs[0].name not in collected.user_requested: - collected.user_requested[reqs[0].name] = i + template = reqs[0] + if ireq.user_supplied and template.name not in collected.user_requested: + collected.user_requested[template.name] = i collected.requirements.extend(reqs) return collected @@ -540,8 +544,14 @@ class Factory: comes_from: Optional[InstallRequirement], requested_extras: Iterable[str] = (), ) -> list[Requirement]: - # TODO: docstring """ + Returns requirement objects associated with the given specifier. In most cases + this will be a single object but the following special cases exist: + - the specifier has markers that do not apply -> result is empty + - the specifier has both a constraint and extras -> result is split + in two requirement objects: one with the constraint and one with the + extra. This allows centralized constraint handling for the base, + resulting in fewer candidate rejections. """ ireq = self._make_install_req_from_spec(specifier, comes_from) return self._make_requirements_from_install_req(ireq, requested_extras) diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index 180158128..31a515da9 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -2,7 +2,7 @@ from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import NormalizedName, canonicalize_name from pip._internal.req.req_install import InstallRequirement -from pip._internal.req.constructors import install_req_without +from pip._internal.req.constructors import install_req_drop_extras from .base import Candidate, CandidateLookup, Requirement, format_name @@ -41,24 +41,20 @@ class ExplicitRequirement(Requirement): class SpecifierRequirement(Requirement): - # TODO: document additional options def __init__( self, ireq: InstallRequirement, *, drop_extras: bool = False, - drop_specifier: bool = False, ) -> None: + """ + :param drop_extras: Ignore any extras that are part of the install requirement, + making this a requirement on the base only. + """ assert ireq.link is None, "This is a link, not a specifier" self._drop_extras: bool = drop_extras - self._extras = frozenset(ireq.extras if not drop_extras else ()) - self._ireq = ( - ireq - if not drop_extras and not drop_specifier - else install_req_without( - ireq, without_extras=self._drop_extras, without_specifier=drop_specifier - ) - ) + self._ireq = ireq if not drop_extras else install_req_drop_extras(ireq) + self._extras = frozenset(self._ireq.extras) def __str__(self) -> str: return str(self._ireq) From cb0f97f70e8b7fbc89e63ed1d2fb5b2dd233fafb Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 22 Jun 2023 15:56:23 +0200 Subject: [PATCH 011/156] reverted troublesome changes --- src/pip/_internal/resolution/resolvelib/factory.py | 11 ++--------- tests/functional/test_install.py | 6 +++--- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 0c2c6ab79..847cbee8d 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -477,9 +477,7 @@ class Factory: self._fail_if_link_is_unsupported_wheel(ireq.link) cand = self._make_candidate_from_link( ireq.link, - # make just the base candidate so the corresponding requirement can be split - # in case of extras (see docstring) - extras=frozenset(), + extras=frozenset(ireq.extras), template=ireq, name=canonicalize_name(ireq.name) if ireq.name else None, version=None, @@ -494,12 +492,7 @@ class Factory: if not ireq.name: raise self._build_failures[ireq.link] return [UnsatisfiableRequirement(canonicalize_name(ireq.name))] - return [ - self.make_requirement_from_candidate(cand), - self.make_requirement_from_candidate( - self._make_extras_candidate(cand, frozenset(ireq.extras), ireq) - ), - ] + return [self.make_requirement_from_candidate(cand)] def collect_root_requirements( self, root_ireqs: List[InstallRequirement] diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index f5ac31a8e..8559d9368 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -2465,6 +2465,6 @@ def test_install_pip_prints_req_chain_pypi(script: PipTestEnvironment) -> None: ) assert ( - "Collecting python-openid " - f"(from Paste[openid]->Paste[openid]==1.7.5.1->-r {req_path} (line 1))" - ) in result.stdout + f"Collecting python-openid " + f"(from Paste[openid]==1.7.5.1->-r {req_path} (line 1))" in result.stdout + ) From 3160293193d947eecb16c2d69d754b04d98b2bab Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 22 Jun 2023 16:10:09 +0200 Subject: [PATCH 012/156] improvement --- src/pip/_internal/resolution/resolvelib/factory.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 847cbee8d..03820edde 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -468,8 +468,6 @@ class Factory: if ireq.extras and ireq.req.specifier: return [ SpecifierRequirement(ireq, drop_extras=True), - # TODO: put this all the way at the back to have even fewer - # candidates? SpecifierRequirement(ireq), ] else: @@ -524,6 +522,15 @@ class Factory: if ireq.user_supplied and template.name not in collected.user_requested: collected.user_requested[template.name] = i collected.requirements.extend(reqs) + # Put requirements with extras at the end of the root requires. This does not + # affect resolvelib's picking preference but it does affect its initial criteria + # population: by putting extras at the end we enable the candidate finder to + # present resolvelib with a smaller set of candidates to resolvelib, already + # taking into account any non-transient constraints on the associated base. This + # means resolvelib will have fewer candidates to visit and reject. + # Python's list sort is stable, meaning relative order is kept for objects with + # the same key. + collected.requirements.sort(key=lambda r: r.name != r.project_name) return collected def make_requirement_from_candidate( From 1038f15496f037181a18e5d67d8a0f33e5cb7cc9 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 22 Jun 2023 16:24:26 +0200 Subject: [PATCH 013/156] stray todo --- src/pip/_internal/resolution/resolvelib/provider.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 121e48d07..315fb9c89 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -184,8 +184,6 @@ class PipProvider(_ProviderBase): # the backtracking backtrack_cause = self.is_backtrack_cause(identifier, backtrack_causes) - # TODO: finally prefer base over extra for the same package - return ( not requires_python, not direct, From c1ead0aa37d5f3526820fcbec1c89011c5063236 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 May 2022 11:14:16 -0400 Subject: [PATCH 014/156] Switch to new cache format and new cache location. --- news/2984.bugfix | 1 + src/pip/_internal/cli/req_command.py | 2 +- src/pip/_internal/commands/cache.py | 21 ++++++++++++------ src/pip/_internal/network/cache.py | 32 +++++++++++++++++++++------- tests/functional/test_cache.py | 4 ++-- 5 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 news/2984.bugfix diff --git a/news/2984.bugfix b/news/2984.bugfix new file mode 100644 index 000000000..d75974349 --- /dev/null +++ b/news/2984.bugfix @@ -0,0 +1 @@ +pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). \ No newline at end of file diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index c2f4e38be..c9c201959 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -123,7 +123,7 @@ class SessionCommandMixin(CommandContextMixIn): ssl_context = None session = PipSession( - cache=os.path.join(cache_dir, "http") if cache_dir else None, + cache=os.path.join(cache_dir, "http-v2") if cache_dir else None, retries=retries if retries is not None else options.retries, trusted_hosts=options.trusted_hosts, index_urls=self._get_index_urls(options), diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index e96d2b492..a11e151f3 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -93,17 +93,21 @@ class CacheCommand(Command): num_http_files = len(self._find_http_files(options)) num_packages = len(self._find_wheels(options, "*")) - http_cache_location = self._cache_dir(options, "http") + http_cache_location = self._cache_dir(options, "http-v2") + old_http_cache_location = self._cache_dir(options, "http") wheels_cache_location = self._cache_dir(options, "wheels") http_cache_size = filesystem.format_directory_size(http_cache_location) + old_http_cache_size = filesystem.format_directory_size(old_http_cache_location) wheels_cache_size = filesystem.format_directory_size(wheels_cache_location) message = ( textwrap.dedent( """ - Package index page cache location: {http_cache_location} - Package index page cache size: {http_cache_size} - Number of HTTP files: {num_http_files} + Package index page cache location (new): {http_cache_location} + Package index page cache location (old): {old_http_cache_location} + Package index page cache size (new): {http_cache_size} + Package index page cache size (old): {old_http_cache_size} + Number of HTTP files (old+new cache): {num_http_files} Locally built wheels location: {wheels_cache_location} Locally built wheels size: {wheels_cache_size} Number of locally built wheels: {package_count} @@ -111,7 +115,9 @@ class CacheCommand(Command): ) .format( http_cache_location=http_cache_location, + old_http_cache_location=old_http_cache_location, http_cache_size=http_cache_size, + old_http_cache_size=old_http_cache_size, num_http_files=num_http_files, wheels_cache_location=wheels_cache_location, package_count=num_packages, @@ -195,8 +201,11 @@ class CacheCommand(Command): return os.path.join(options.cache_dir, subdir) def _find_http_files(self, options: Values) -> List[str]: - http_dir = self._cache_dir(options, "http") - return filesystem.find_files(http_dir, "*") + old_http_dir = self._cache_dir(options, "http") + new_http_dir = self._cache_dir(options, "http-v2") + return filesystem.find_files(old_http_dir, "*") + filesystem.find_files( + new_http_dir, "*" + ) def _find_wheels(self, options: Values, pattern: str) -> List[str]: wheel_dir = self._cache_dir(options, "wheels") diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index a81a23985..b85be2e48 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -3,10 +3,10 @@ import os from contextlib import contextmanager -from typing import Generator, Optional +from typing import BinaryIO, Generator, Optional -from pip._vendor.cachecontrol.cache import BaseCache -from pip._vendor.cachecontrol.caches import FileCache +from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache +from pip._vendor.cachecontrol.caches import SeparateBodyFileCache from pip._vendor.requests.models import Response from pip._internal.utils.filesystem import adjacent_tmp_file, replace @@ -28,7 +28,7 @@ def suppressed_cache_errors() -> Generator[None, None, None]: pass -class SafeFileCache(BaseCache): +class SafeFileCache(SeparateBodyBaseCache): """ A file based cache which is safe to use even when the target directory may not be accessible or writable. @@ -43,7 +43,7 @@ class SafeFileCache(BaseCache): # From cachecontrol.caches.file_cache.FileCache._fn, brought into our # class for backwards-compatibility and to avoid using a non-public # method. - hashed = FileCache.encode(name) + hashed = SeparateBodyFileCache.encode(name) parts = list(hashed[:5]) + [hashed] return os.path.join(self.directory, *parts) @@ -53,17 +53,33 @@ class SafeFileCache(BaseCache): with open(path, "rb") as f: return f.read() - def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None: - path = self._get_cache_path(key) + def _write(self, path: str, data: bytes) -> None: with suppressed_cache_errors(): ensure_dir(os.path.dirname(path)) with adjacent_tmp_file(path) as f: - f.write(value) + f.write(data) replace(f.name, path) + def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None: + path = self._get_cache_path(key) + self._write(path, value) + def delete(self, key: str) -> None: path = self._get_cache_path(key) with suppressed_cache_errors(): os.remove(path) + os.remove(path + ".body") + + def get_body(self, key: str) -> Optional[BinaryIO]: + path = self._get_cache_path(key) + ".body" + with suppressed_cache_errors(): + return open(path, "rb") + + def set_body(self, key: str, body: Optional[bytes]) -> None: + if body is None: + # Workaround for https://github.com/ionrock/cachecontrol/issues/276 + return + path = self._get_cache_path(key) + ".body" + self._write(path, body) diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index 788abdd2b..5eea6a96e 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -20,7 +20,7 @@ def cache_dir(script: PipTestEnvironment) -> str: @pytest.fixture def http_cache_dir(cache_dir: str) -> str: - return os.path.normcase(os.path.join(cache_dir, "http")) + return os.path.normcase(os.path.join(cache_dir, "http-v2")) @pytest.fixture @@ -211,7 +211,7 @@ def test_cache_info( ) -> None: result = script.pip("cache", "info") - assert f"Package index page cache location: {http_cache_dir}" in result.stdout + assert f"Package index page cache location (new): {http_cache_dir}" in result.stdout assert f"Locally built wheels location: {wheel_cache_dir}" in result.stdout num_wheels = len(wheel_cache_files) assert f"Number of locally built wheels: {num_wheels}" in result.stdout From fa87c9eb23dd25ad5cb03fe480a3fc4b92deb7a6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 May 2022 13:06:38 -0400 Subject: [PATCH 015/156] Testing for body methods of network cache. --- src/pip/_internal/network/cache.py | 1 + tests/unit/test_network_cache.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index b85be2e48..11c76bf0f 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -70,6 +70,7 @@ class SafeFileCache(SeparateBodyBaseCache): path = self._get_cache_path(key) with suppressed_cache_errors(): os.remove(path) + with suppressed_cache_errors(): os.remove(path + ".body") def get_body(self, key: str) -> Optional[BinaryIO]: diff --git a/tests/unit/test_network_cache.py b/tests/unit/test_network_cache.py index a5519864f..88597e4c1 100644 --- a/tests/unit/test_network_cache.py +++ b/tests/unit/test_network_cache.py @@ -31,6 +31,14 @@ class TestSafeFileCache: cache.delete("test key") assert cache.get("test key") is None + def test_cache_roundtrip_body(self, cache_tmpdir: Path) -> None: + cache = SafeFileCache(os.fspath(cache_tmpdir)) + assert cache.get_body("test key") is None + cache.set_body("test key", b"a test string") + assert cache.get_body("test key").read() == b"a test string" + cache.delete("test key") + assert cache.get_body("test key") is None + @pytest.mark.skipif("sys.platform == 'win32'") def test_safe_get_no_perms( self, cache_tmpdir: Path, monkeypatch: pytest.MonkeyPatch From fde34fdf8416a9692c07a899d2668f3f6ccf9df7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 12:06:15 -0400 Subject: [PATCH 016/156] Temporary workaround for https://github.com/ionrock/cachecontrol/issues/276 until it's fixed upstream. --- src/pip/_vendor/cachecontrol/controller.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 7f23529f1..14ba62976 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -407,7 +407,17 @@ class CacheController(object): """ cache_url = self.cache_url(request.url) - cached_response = self.serializer.loads(request, self.cache.get(cache_url)) + # NOTE: This is a hot-patch for + # https://github.com/ionrock/cachecontrol/issues/276 until it's fixed + # upstream. + if isinstance(self.cache, SeparateBodyBaseCache): + body_file = self.cache.get_body(cache_url) + else: + body_file = None + + cached_response = self.serializer.loads( + request, self.cache.get(cache_url), body_file + ) if not cached_response: # we didn't have a cached response From 5b7c999581e1b892a8048f6bd1275e8501614911 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 12:10:05 -0400 Subject: [PATCH 017/156] Whitespace fix. --- news/2984.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/2984.bugfix b/news/2984.bugfix index d75974349..cce561815 100644 --- a/news/2984.bugfix +++ b/news/2984.bugfix @@ -1 +1 @@ -pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). \ No newline at end of file +pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). From 7a609bfdd5a23d404124a0ace5e3598966fe2466 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 12:11:34 -0400 Subject: [PATCH 018/156] Mypy fix. --- tests/unit/test_network_cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_network_cache.py b/tests/unit/test_network_cache.py index 88597e4c1..d62d1ab69 100644 --- a/tests/unit/test_network_cache.py +++ b/tests/unit/test_network_cache.py @@ -35,7 +35,9 @@ class TestSafeFileCache: cache = SafeFileCache(os.fspath(cache_tmpdir)) assert cache.get_body("test key") is None cache.set_body("test key", b"a test string") - assert cache.get_body("test key").read() == b"a test string" + body = cache.get_body("test key") + assert body is not None + assert body.read() == b"a test string" cache.delete("test key") assert cache.get_body("test key") is None From 3dbba12132b55a937c095c1c5537baf8652533ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 12:12:29 -0400 Subject: [PATCH 019/156] Correct name. --- news/{2984.bugfix => 2984.bugfix.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{2984.bugfix => 2984.bugfix.rst} (100%) diff --git a/news/2984.bugfix b/news/2984.bugfix.rst similarity index 100% rename from news/2984.bugfix rename to news/2984.bugfix.rst From bff05e5622b1dcf66c1556fb421441086b93456c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 14:50:29 -0400 Subject: [PATCH 020/156] Switch to proposed upstream fix. --- src/pip/_internal/network/cache.py | 3 -- src/pip/_vendor/cachecontrol/controller.py | 57 +++++++++++----------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index 11c76bf0f..f52e9974f 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -79,8 +79,5 @@ class SafeFileCache(SeparateBodyBaseCache): return open(path, "rb") def set_body(self, key: str, body: Optional[bytes]) -> None: - if body is None: - # Workaround for https://github.com/ionrock/cachecontrol/issues/276 - return path = self._get_cache_path(key) + ".body" self._write(path, body) diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 14ba62976..7af0e002d 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -122,6 +122,26 @@ class CacheController(object): return retval + def _load_from_cache(self, request): + """ + Load a cached response, or return None if it's not available. + """ + cache_url = request.url + cache_data = self.cache.get(cache_url) + if cache_data is None: + logger.debug("No cache entry available") + return None + + if isinstance(self.cache, SeparateBodyBaseCache): + body_file = self.cache.get_body(cache_url) + else: + body_file = None + + result = self.serializer.loads(request, cache_data, body_file) + if result is None: + logger.warning("Cache entry deserialization failed, entry ignored") + return result + def cached_request(self, request): """ Return a cached response if it exists in the cache, otherwise @@ -140,21 +160,9 @@ class CacheController(object): logger.debug('Request header has "max_age" as 0, cache bypassed') return False - # Request allows serving from the cache, let's see if we find something - cache_data = self.cache.get(cache_url) - if cache_data is None: - logger.debug("No cache entry available") - return False - - if isinstance(self.cache, SeparateBodyBaseCache): - body_file = self.cache.get_body(cache_url) - else: - body_file = None - - # Check whether it can be deserialized - resp = self.serializer.loads(request, cache_data, body_file) + # Check whether we can load the response from the cache: + resp = self._load_from_cache(request) if not resp: - logger.warning("Cache entry deserialization failed, entry ignored") return False # If we have a cached permanent redirect, return it immediately. We @@ -240,8 +248,7 @@ class CacheController(object): return False def conditional_headers(self, request): - cache_url = self.cache_url(request.url) - resp = self.serializer.loads(request, self.cache.get(cache_url)) + resp = self._load_from_cache(request) new_headers = {} if resp: @@ -267,7 +274,10 @@ class CacheController(object): self.serializer.dumps(request, response, b""), expires=expires_time, ) - self.cache.set_body(cache_url, body) + # body is None can happen when, for example, we're only updating + # headers, as is the case in update_cached_response(). + if body is not None: + self.cache.set_body(cache_url, body) else: self.cache.set( cache_url, @@ -406,18 +416,7 @@ class CacheController(object): gotten a 304 as the response. """ cache_url = self.cache_url(request.url) - - # NOTE: This is a hot-patch for - # https://github.com/ionrock/cachecontrol/issues/276 until it's fixed - # upstream. - if isinstance(self.cache, SeparateBodyBaseCache): - body_file = self.cache.get_body(cache_url) - else: - body_file = None - - cached_response = self.serializer.loads( - request, self.cache.get(cache_url), body_file - ) + cached_response = self._load_from_cache(request) if not cached_response: # we didn't have a cached response From 46f9154daecffa52966f6e917f0819cfabb112ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 15:11:16 -0400 Subject: [PATCH 021/156] Make sure the file gets closed. --- tests/unit/test_network_cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_network_cache.py b/tests/unit/test_network_cache.py index d62d1ab69..aa849f3b0 100644 --- a/tests/unit/test_network_cache.py +++ b/tests/unit/test_network_cache.py @@ -37,7 +37,8 @@ class TestSafeFileCache: cache.set_body("test key", b"a test string") body = cache.get_body("test key") assert body is not None - assert body.read() == b"a test string" + with body: + assert body.read() == b"a test string" cache.delete("test key") assert cache.get_body("test key") is None From bada6316dfcb16d50f214b88f8d2424f0e9d990b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 15:13:46 -0400 Subject: [PATCH 022/156] More accurate type. --- src/pip/_internal/network/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index f52e9974f..d6b8ccdcf 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -78,6 +78,6 @@ class SafeFileCache(SeparateBodyBaseCache): with suppressed_cache_errors(): return open(path, "rb") - def set_body(self, key: str, body: Optional[bytes]) -> None: + def set_body(self, key: str, body: bytes) -> None: path = self._get_cache_path(key) + ".body" self._write(path, body) From ca08c16b9e81ce21021831fb1bdfe3a76387fd25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Jun 2023 13:56:56 -0400 Subject: [PATCH 023/156] Vendor latest version of CacheControl. --- src/pip/_vendor/cachecontrol.pyi | 1 - src/pip/_vendor/cachecontrol/__init__.py | 18 ++- src/pip/_vendor/cachecontrol/_cmd.py | 24 ++-- src/pip/_vendor/cachecontrol/adapter.py | 83 +++++++---- src/pip/_vendor/cachecontrol/cache.py | 31 ++-- .../_vendor/cachecontrol/caches/__init__.py | 5 +- .../_vendor/cachecontrol/caches/file_cache.py | 78 +++++----- .../cachecontrol/caches/redis_cache.py | 29 ++-- src/pip/_vendor/cachecontrol/compat.py | 32 ----- src/pip/_vendor/cachecontrol/controller.py | 116 ++++++++++----- src/pip/_vendor/cachecontrol/filewrapper.py | 27 ++-- src/pip/_vendor/cachecontrol/heuristics.py | 54 ++++--- src/pip/_vendor/cachecontrol/py.typed | 0 src/pip/_vendor/cachecontrol/serialize.py | 135 ++++++++++++------ src/pip/_vendor/cachecontrol/wrapper.py | 33 +++-- src/pip/_vendor/vendor.txt | 2 +- 16 files changed, 405 insertions(+), 263 deletions(-) delete mode 100644 src/pip/_vendor/cachecontrol.pyi delete mode 100644 src/pip/_vendor/cachecontrol/compat.py create mode 100644 src/pip/_vendor/cachecontrol/py.typed diff --git a/src/pip/_vendor/cachecontrol.pyi b/src/pip/_vendor/cachecontrol.pyi deleted file mode 100644 index 636a66bac..000000000 --- a/src/pip/_vendor/cachecontrol.pyi +++ /dev/null @@ -1 +0,0 @@ -from cachecontrol import * \ No newline at end of file diff --git a/src/pip/_vendor/cachecontrol/__init__.py b/src/pip/_vendor/cachecontrol/__init__.py index f631ae6df..3701cdd6b 100644 --- a/src/pip/_vendor/cachecontrol/__init__.py +++ b/src/pip/_vendor/cachecontrol/__init__.py @@ -8,11 +8,21 @@ Make it easy to import from cachecontrol without long namespaces. """ __author__ = "Eric Larson" __email__ = "eric@ionrock.org" -__version__ = "0.12.11" +__version__ = "0.13.0" -from .wrapper import CacheControl -from .adapter import CacheControlAdapter -from .controller import CacheController +from pip._vendor.cachecontrol.adapter import CacheControlAdapter +from pip._vendor.cachecontrol.controller import CacheController +from pip._vendor.cachecontrol.wrapper import CacheControl + +__all__ = [ + "__author__", + "__email__", + "__version__", + "CacheControlAdapter", + "CacheController", + "CacheControl", +] import logging + logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/src/pip/_vendor/cachecontrol/_cmd.py b/src/pip/_vendor/cachecontrol/_cmd.py index 4266b5ee9..ab4dac3dd 100644 --- a/src/pip/_vendor/cachecontrol/_cmd.py +++ b/src/pip/_vendor/cachecontrol/_cmd.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 import logging +from argparse import ArgumentParser +from typing import TYPE_CHECKING from pip._vendor import requests @@ -10,16 +12,19 @@ from pip._vendor.cachecontrol.adapter import CacheControlAdapter from pip._vendor.cachecontrol.cache import DictCache from pip._vendor.cachecontrol.controller import logger -from argparse import ArgumentParser +if TYPE_CHECKING: + from argparse import Namespace + + from pip._vendor.cachecontrol.controller import CacheController -def setup_logging(): +def setup_logging() -> None: logger.setLevel(logging.DEBUG) handler = logging.StreamHandler() logger.addHandler(handler) -def get_session(): +def get_session() -> requests.Session: adapter = CacheControlAdapter( DictCache(), cache_etags=True, serializer=None, heuristic=None ) @@ -27,17 +32,17 @@ def get_session(): sess.mount("http://", adapter) sess.mount("https://", adapter) - sess.cache_controller = adapter.controller + sess.cache_controller = adapter.controller # type: ignore[attr-defined] return sess -def get_args(): +def get_args() -> "Namespace": parser = ArgumentParser() parser.add_argument("url", help="The URL to try and cache") return parser.parse_args() -def main(args=None): +def main() -> None: args = get_args() sess = get_session() @@ -48,10 +53,13 @@ def main(args=None): setup_logging() # try setting the cache - sess.cache_controller.cache_response(resp.request, resp.raw) + cache_controller: "CacheController" = ( + sess.cache_controller # type: ignore[attr-defined] + ) + cache_controller.cache_response(resp.request, resp.raw) # Now try to get it - if sess.cache_controller.cached_request(resp.request): + if cache_controller.cached_request(resp.request): print("Cached!") else: print("Not cached :(") diff --git a/src/pip/_vendor/cachecontrol/adapter.py b/src/pip/_vendor/cachecontrol/adapter.py index 94c75e1a0..83c08e003 100644 --- a/src/pip/_vendor/cachecontrol/adapter.py +++ b/src/pip/_vendor/cachecontrol/adapter.py @@ -2,15 +2,24 @@ # # SPDX-License-Identifier: Apache-2.0 -import types import functools +import types import zlib +from typing import TYPE_CHECKING, Any, Collection, Mapping, Optional, Tuple, Type, Union from pip._vendor.requests.adapters import HTTPAdapter -from .controller import CacheController, PERMANENT_REDIRECT_STATUSES -from .cache import DictCache -from .filewrapper import CallbackFileWrapper +from pip._vendor.cachecontrol.cache import DictCache +from pip._vendor.cachecontrol.controller import PERMANENT_REDIRECT_STATUSES, CacheController +from pip._vendor.cachecontrol.filewrapper import CallbackFileWrapper + +if TYPE_CHECKING: + from pip._vendor.requests import PreparedRequest, Response + from pip._vendor.urllib3 import HTTPResponse + + from pip._vendor.cachecontrol.cache import BaseCache + from pip._vendor.cachecontrol.heuristics import BaseHeuristic + from pip._vendor.cachecontrol.serialize import Serializer class CacheControlAdapter(HTTPAdapter): @@ -18,15 +27,15 @@ class CacheControlAdapter(HTTPAdapter): def __init__( self, - cache=None, - cache_etags=True, - controller_class=None, - serializer=None, - heuristic=None, - cacheable_methods=None, - *args, - **kw - ): + cache: Optional["BaseCache"] = None, + cache_etags: bool = True, + controller_class: Optional[Type[CacheController]] = None, + serializer: Optional["Serializer"] = None, + heuristic: Optional["BaseHeuristic"] = None, + cacheable_methods: Optional[Collection[str]] = None, + *args: Any, + **kw: Any, + ) -> None: super(CacheControlAdapter, self).__init__(*args, **kw) self.cache = DictCache() if cache is None else cache self.heuristic = heuristic @@ -37,7 +46,18 @@ class CacheControlAdapter(HTTPAdapter): self.cache, cache_etags=cache_etags, serializer=serializer ) - def send(self, request, cacheable_methods=None, **kw): + def send( + self, + request: "PreparedRequest", + stream: bool = False, + timeout: Union[None, float, Tuple[float, float], Tuple[float, None]] = None, + verify: Union[bool, str] = True, + cert: Union[ + None, bytes, str, Tuple[Union[bytes, str], Union[bytes, str]] + ] = None, + proxies: Optional[Mapping[str, str]] = None, + cacheable_methods: Optional[Collection[str]] = None, + ) -> "Response": """ Send a request. Use the request information to see if it exists in the cache and cache the response if we need to and can. @@ -54,13 +74,19 @@ class CacheControlAdapter(HTTPAdapter): # check for etags and add headers if appropriate request.headers.update(self.controller.conditional_headers(request)) - resp = super(CacheControlAdapter, self).send(request, **kw) + resp = super(CacheControlAdapter, self).send( + request, stream, timeout, verify, cert, proxies + ) return resp def build_response( - self, request, response, from_cache=False, cacheable_methods=None - ): + self, + request: "PreparedRequest", + response: "HTTPResponse", + from_cache: bool = False, + cacheable_methods: Optional[Collection[str]] = None, + ) -> "Response": """ Build a response by making a request or using the cache. @@ -102,36 +128,39 @@ class CacheControlAdapter(HTTPAdapter): else: # Wrap the response file with a wrapper that will cache the # response when the stream has been consumed. - response._fp = CallbackFileWrapper( - response._fp, + response._fp = CallbackFileWrapper( # type: ignore[attr-defined] + response._fp, # type: ignore[attr-defined] functools.partial( self.controller.cache_response, request, response ), ) if response.chunked: - super_update_chunk_length = response._update_chunk_length + super_update_chunk_length = response._update_chunk_length # type: ignore[attr-defined] - def _update_chunk_length(self): + def _update_chunk_length(self: "HTTPResponse") -> None: super_update_chunk_length() if self.chunk_left == 0: - self._fp._close() + self._fp._close() # type: ignore[attr-defined] - response._update_chunk_length = types.MethodType( + response._update_chunk_length = types.MethodType( # type: ignore[attr-defined] _update_chunk_length, response ) - resp = super(CacheControlAdapter, self).build_response(request, response) + resp: "Response" = super( # type: ignore[no-untyped-call] + CacheControlAdapter, self + ).build_response(request, response) # See if we should invalidate the cache. if request.method in self.invalidating_methods and resp.ok: + assert request.url is not None cache_url = self.controller.cache_url(request.url) self.cache.delete(cache_url) # Give the request a from_cache attr to let people use it - resp.from_cache = from_cache + resp.from_cache = from_cache # type: ignore[attr-defined] return resp - def close(self): + def close(self) -> None: self.cache.close() - super(CacheControlAdapter, self).close() + super(CacheControlAdapter, self).close() # type: ignore[no-untyped-call] diff --git a/src/pip/_vendor/cachecontrol/cache.py b/src/pip/_vendor/cachecontrol/cache.py index 2a965f595..61031d234 100644 --- a/src/pip/_vendor/cachecontrol/cache.py +++ b/src/pip/_vendor/cachecontrol/cache.py @@ -7,37 +7,43 @@ The cache object API for implementing caches. The default is a thread safe in-memory dictionary. """ from threading import Lock +from typing import IO, TYPE_CHECKING, MutableMapping, Optional, Union + +if TYPE_CHECKING: + from datetime import datetime class BaseCache(object): - - def get(self, key): + def get(self, key: str) -> Optional[bytes]: raise NotImplementedError() - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + ) -> None: raise NotImplementedError() - def delete(self, key): + def delete(self, key: str) -> None: raise NotImplementedError() - def close(self): + def close(self) -> None: pass class DictCache(BaseCache): - - def __init__(self, init_dict=None): + def __init__(self, init_dict: Optional[MutableMapping[str, bytes]] = None) -> None: self.lock = Lock() self.data = init_dict or {} - def get(self, key): + def get(self, key: str) -> Optional[bytes]: return self.data.get(key, None) - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + ) -> None: with self.lock: self.data.update({key: value}) - def delete(self, key): + def delete(self, key: str) -> None: with self.lock: if key in self.data: self.data.pop(key) @@ -55,10 +61,11 @@ class SeparateBodyBaseCache(BaseCache): Similarly, the body should be loaded separately via ``get_body()``. """ - def set_body(self, key, body): + + def set_body(self, key: str, body: bytes) -> None: raise NotImplementedError() - def get_body(self, key): + def get_body(self, key: str) -> Optional["IO[bytes]"]: """ Return the body as file-like object. """ diff --git a/src/pip/_vendor/cachecontrol/caches/__init__.py b/src/pip/_vendor/cachecontrol/caches/__init__.py index 37827291f..24ff469ff 100644 --- a/src/pip/_vendor/cachecontrol/caches/__init__.py +++ b/src/pip/_vendor/cachecontrol/caches/__init__.py @@ -2,8 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from .file_cache import FileCache, SeparateBodyFileCache -from .redis_cache import RedisCache - +from pip._vendor.cachecontrol.caches.file_cache import FileCache, SeparateBodyFileCache +from pip._vendor.cachecontrol.caches.redis_cache import RedisCache __all__ = ["FileCache", "SeparateBodyFileCache", "RedisCache"] diff --git a/src/pip/_vendor/cachecontrol/caches/file_cache.py b/src/pip/_vendor/cachecontrol/caches/file_cache.py index f1ddb2ebd..0437c4e8a 100644 --- a/src/pip/_vendor/cachecontrol/caches/file_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/file_cache.py @@ -5,18 +5,18 @@ import hashlib import os from textwrap import dedent +from typing import IO, TYPE_CHECKING, Optional, Type, Union -from ..cache import BaseCache, SeparateBodyBaseCache -from ..controller import CacheController +from pip._vendor.cachecontrol.cache import BaseCache, SeparateBodyBaseCache +from pip._vendor.cachecontrol.controller import CacheController -try: - FileNotFoundError -except NameError: - # py2.X - FileNotFoundError = (IOError, OSError) +if TYPE_CHECKING: + from datetime import datetime + + from filelock import BaseFileLock -def _secure_open_write(filename, fmode): +def _secure_open_write(filename: str, fmode: int) -> "IO[bytes]": # We only want to write to this file, so open it in write only mode flags = os.O_WRONLY @@ -62,37 +62,27 @@ class _FileCacheMixin: def __init__( self, - directory, - forever=False, - filemode=0o0600, - dirmode=0o0700, - use_dir_lock=None, - lock_class=None, - ): - - if use_dir_lock is not None and lock_class is not None: - raise ValueError("Cannot use use_dir_lock and lock_class together") - + directory: str, + forever: bool = False, + filemode: int = 0o0600, + dirmode: int = 0o0700, + lock_class: Optional[Type["BaseFileLock"]] = None, + ) -> None: try: - from lockfile import LockFile - from lockfile.mkdirlockfile import MkdirLockFile + if lock_class is None: + from filelock import FileLock + + lock_class = FileLock except ImportError: notice = dedent( """ NOTE: In order to use the FileCache you must have - lockfile installed. You can install it via pip: - pip install lockfile + filelock installed. You can install it via pip: + pip install filelock """ ) raise ImportError(notice) - else: - if use_dir_lock: - lock_class = MkdirLockFile - - elif lock_class is None: - lock_class = LockFile - self.directory = directory self.forever = forever self.filemode = filemode @@ -100,17 +90,17 @@ class _FileCacheMixin: self.lock_class = lock_class @staticmethod - def encode(x): + def encode(x: str) -> str: return hashlib.sha224(x.encode()).hexdigest() - def _fn(self, name): + def _fn(self, name: str) -> str: # NOTE: This method should not change as some may depend on it. # See: https://github.com/ionrock/cachecontrol/issues/63 hashed = self.encode(name) parts = list(hashed[:5]) + [hashed] return os.path.join(self.directory, *parts) - def get(self, key): + def get(self, key: str) -> Optional[bytes]: name = self._fn(key) try: with open(name, "rb") as fh: @@ -119,11 +109,13 @@ class _FileCacheMixin: except FileNotFoundError: return None - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + ) -> None: name = self._fn(key) self._write(name, value) - def _write(self, path, data: bytes): + def _write(self, path: str, data: bytes) -> None: """ Safely write the data to the given path. """ @@ -133,12 +125,12 @@ class _FileCacheMixin: except (IOError, OSError): pass - with self.lock_class(path) as lock: + with self.lock_class(path + ".lock"): # Write our actual file - with _secure_open_write(lock.path, self.filemode) as fh: + with _secure_open_write(path, self.filemode) as fh: fh.write(data) - def _delete(self, key, suffix): + def _delete(self, key: str, suffix: str) -> None: name = self._fn(key) + suffix if not self.forever: try: @@ -153,7 +145,7 @@ class FileCache(_FileCacheMixin, BaseCache): downloads. """ - def delete(self, key): + def delete(self, key: str) -> None: self._delete(key, "") @@ -163,23 +155,23 @@ class SeparateBodyFileCache(_FileCacheMixin, SeparateBodyBaseCache): peak memory usage. """ - def get_body(self, key): + def get_body(self, key: str) -> Optional["IO[bytes]"]: name = self._fn(key) + ".body" try: return open(name, "rb") except FileNotFoundError: return None - def set_body(self, key, body): + def set_body(self, key: str, body: bytes) -> None: name = self._fn(key) + ".body" self._write(name, body) - def delete(self, key): + def delete(self, key: str) -> None: self._delete(key, "") self._delete(key, ".body") -def url_to_file_path(url, filecache): +def url_to_file_path(url: str, filecache: FileCache) -> str: """Return the file cache path based on the URL. This does not ensure the file exists! diff --git a/src/pip/_vendor/cachecontrol/caches/redis_cache.py b/src/pip/_vendor/cachecontrol/caches/redis_cache.py index 2cba4b070..f7ae45d38 100644 --- a/src/pip/_vendor/cachecontrol/caches/redis_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/redis_cache.py @@ -4,36 +4,45 @@ from __future__ import division -from datetime import datetime +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional, Union + from pip._vendor.cachecontrol.cache import BaseCache +if TYPE_CHECKING: + from redis import Redis + class RedisCache(BaseCache): - - def __init__(self, conn): + def __init__(self, conn: "Redis[bytes]") -> None: self.conn = conn - def get(self, key): + def get(self, key: str) -> Optional[bytes]: return self.conn.get(key) - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: Optional[Union[int, datetime]] = None + ) -> None: if not expires: self.conn.set(key, value) elif isinstance(expires, datetime): - expires = expires - datetime.utcnow() - self.conn.setex(key, int(expires.total_seconds()), value) + now_utc = datetime.now(timezone.utc) + if expires.tzinfo is None: + now_utc = now_utc.replace(tzinfo=None) + delta = expires - now_utc + self.conn.setex(key, int(delta.total_seconds()), value) else: self.conn.setex(key, expires, value) - def delete(self, key): + def delete(self, key: str) -> None: self.conn.delete(key) - def clear(self): + def clear(self) -> None: """Helper for clearing all the keys in a database. Use with caution!""" for key in self.conn.keys(): self.conn.delete(key) - def close(self): + def close(self) -> None: """Redis uses connection pooling, no need to close the connection.""" pass diff --git a/src/pip/_vendor/cachecontrol/compat.py b/src/pip/_vendor/cachecontrol/compat.py deleted file mode 100644 index ccec9379d..000000000 --- a/src/pip/_vendor/cachecontrol/compat.py +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: 2015 Eric Larson -# -# SPDX-License-Identifier: Apache-2.0 - -try: - from urllib.parse import urljoin -except ImportError: - from urlparse import urljoin - - -try: - import cPickle as pickle -except ImportError: - import pickle - -# Handle the case where the requests module has been patched to not have -# urllib3 bundled as part of its source. -try: - from pip._vendor.requests.packages.urllib3.response import HTTPResponse -except ImportError: - from pip._vendor.urllib3.response import HTTPResponse - -try: - from pip._vendor.requests.packages.urllib3.util import is_fp_closed -except ImportError: - from pip._vendor.urllib3.util import is_fp_closed - -# Replicate some six behaviour -try: - text_type = unicode -except NameError: - text_type = str diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 7af0e002d..3365d9621 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -5,17 +5,25 @@ """ The httplib2 algorithms ported for use with requests. """ +import calendar import logging import re -import calendar import time from email.utils import parsedate_tz +from typing import TYPE_CHECKING, Collection, Dict, Mapping, Optional, Tuple, Union from pip._vendor.requests.structures import CaseInsensitiveDict -from .cache import DictCache, SeparateBodyBaseCache -from .serialize import Serializer +from pip._vendor.cachecontrol.cache import DictCache, SeparateBodyBaseCache +from pip._vendor.cachecontrol.serialize import Serializer +if TYPE_CHECKING: + from typing import Literal + + from pip._vendor.requests import PreparedRequest + from pip._vendor.urllib3 import HTTPResponse + + from pip._vendor.cachecontrol.cache import BaseCache logger = logging.getLogger(__name__) @@ -24,12 +32,14 @@ URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") PERMANENT_REDIRECT_STATUSES = (301, 308) -def parse_uri(uri): +def parse_uri(uri: str) -> Tuple[str, str, str, str, str]: """Parses a URI using the regex given in Appendix B of RFC 3986. (scheme, authority, path, query, fragment) = parse_uri(uri) """ - groups = URI.match(uri).groups() + match = URI.match(uri) + assert match is not None + groups = match.groups() return (groups[1], groups[3], groups[4], groups[6], groups[8]) @@ -37,7 +47,11 @@ class CacheController(object): """An interface to see if request should cached or not.""" def __init__( - self, cache=None, cache_etags=True, serializer=None, status_codes=None + self, + cache: Optional["BaseCache"] = None, + cache_etags: bool = True, + serializer: Optional[Serializer] = None, + status_codes: Optional[Collection[int]] = None, ): self.cache = DictCache() if cache is None else cache self.cache_etags = cache_etags @@ -45,7 +59,7 @@ class CacheController(object): self.cacheable_status_codes = status_codes or (200, 203, 300, 301, 308) @classmethod - def _urlnorm(cls, uri): + def _urlnorm(cls, uri: str) -> str: """Normalize the URL to create a safe key for the cache""" (scheme, authority, path, query, fragment) = parse_uri(uri) if not scheme or not authority: @@ -65,10 +79,12 @@ class CacheController(object): return defrag_uri @classmethod - def cache_url(cls, uri): + def cache_url(cls, uri: str) -> str: return cls._urlnorm(uri) - def parse_cache_control(self, headers): + def parse_cache_control( + self, headers: Mapping[str, str] + ) -> Dict[str, Optional[int]]: known_directives = { # https://tools.ietf.org/html/rfc7234#section-5.2 "max-age": (int, True), @@ -87,7 +103,7 @@ class CacheController(object): cc_headers = headers.get("cache-control", headers.get("Cache-Control", "")) - retval = {} + retval: Dict[str, Optional[int]] = {} for cc_directive in cc_headers.split(","): if not cc_directive.strip(): @@ -122,11 +138,12 @@ class CacheController(object): return retval - def _load_from_cache(self, request): + def _load_from_cache(self, request: "PreparedRequest") -> Optional["HTTPResponse"]: """ Load a cached response, or return None if it's not available. """ cache_url = request.url + assert cache_url is not None cache_data = self.cache.get(cache_url) if cache_data is None: logger.debug("No cache entry available") @@ -142,11 +159,14 @@ class CacheController(object): logger.warning("Cache entry deserialization failed, entry ignored") return result - def cached_request(self, request): + def cached_request( + self, request: "PreparedRequest" + ) -> Union["HTTPResponse", "Literal[False]"]: """ Return a cached response if it exists in the cache, otherwise return False. """ + assert request.url is not None cache_url = self.cache_url(request.url) logger.debug('Looking up "%s" in the cache', cache_url) cc = self.parse_cache_control(request.headers) @@ -182,7 +202,7 @@ class CacheController(object): logger.debug(msg) return resp - headers = CaseInsensitiveDict(resp.headers) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers) if not headers or "date" not in headers: if "etag" not in headers: # Without date or etag, the cached response can never be used @@ -193,7 +213,9 @@ class CacheController(object): return False now = time.time() - date = calendar.timegm(parsedate_tz(headers["date"])) + time_tuple = parsedate_tz(headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) current_age = max(0, now - date) logger.debug("Current age based on date: %i", current_age) @@ -207,28 +229,30 @@ class CacheController(object): freshness_lifetime = 0 # Check the max-age pragma in the cache control header - if "max-age" in resp_cc: - freshness_lifetime = resp_cc["max-age"] + max_age = resp_cc.get("max-age") + if max_age is not None: + freshness_lifetime = max_age logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime) # If there isn't a max-age, check for an expires header elif "expires" in headers: expires = parsedate_tz(headers["expires"]) if expires is not None: - expire_time = calendar.timegm(expires) - date + expire_time = calendar.timegm(expires[:6]) - date freshness_lifetime = max(0, expire_time) logger.debug("Freshness lifetime from expires: %i", freshness_lifetime) # Determine if we are setting freshness limit in the # request. Note, this overrides what was in the response. - if "max-age" in cc: - freshness_lifetime = cc["max-age"] + max_age = cc.get("max-age") + if max_age is not None: + freshness_lifetime = max_age logger.debug( "Freshness lifetime from request max-age: %i", freshness_lifetime ) - if "min-fresh" in cc: - min_fresh = cc["min-fresh"] + min_fresh = cc.get("min-fresh") + if min_fresh is not None: # adjust our current age by our min fresh current_age += min_fresh logger.debug("Adjusted current age from min-fresh: %i", current_age) @@ -247,12 +271,12 @@ class CacheController(object): # return the original handler return False - def conditional_headers(self, request): + def conditional_headers(self, request: "PreparedRequest") -> Dict[str, str]: resp = self._load_from_cache(request) new_headers = {} if resp: - headers = CaseInsensitiveDict(resp.headers) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers) if "etag" in headers: new_headers["If-None-Match"] = headers["ETag"] @@ -262,7 +286,14 @@ class CacheController(object): return new_headers - def _cache_set(self, cache_url, request, response, body=None, expires_time=None): + def _cache_set( + self, + cache_url: str, + request: "PreparedRequest", + response: "HTTPResponse", + body: Optional[bytes] = None, + expires_time: Optional[int] = None, + ) -> None: """ Store the data in the cache. """ @@ -285,7 +316,13 @@ class CacheController(object): expires=expires_time, ) - def cache_response(self, request, response, body=None, status_codes=None): + def cache_response( + self, + request: "PreparedRequest", + response: "HTTPResponse", + body: Optional[bytes] = None, + status_codes: Optional[Collection[int]] = None, + ) -> None: """ Algorithm for caching requests. @@ -300,10 +337,14 @@ class CacheController(object): ) return - response_headers = CaseInsensitiveDict(response.headers) + response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + response.headers + ) if "date" in response_headers: - date = calendar.timegm(parsedate_tz(response_headers["date"])) + time_tuple = parsedate_tz(response_headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) else: date = 0 @@ -322,6 +363,7 @@ class CacheController(object): cc_req = self.parse_cache_control(request.headers) cc = self.parse_cache_control(response_headers) + assert request.url is not None cache_url = self.cache_url(request.url) logger.debug('Updating cache with response from "%s"', cache_url) @@ -354,7 +396,7 @@ class CacheController(object): if response_headers.get("expires"): expires = parsedate_tz(response_headers["expires"]) if expires is not None: - expires_time = calendar.timegm(expires) - date + expires_time = calendar.timegm(expires[:6]) - date expires_time = max(expires_time, 14 * 86400) @@ -372,11 +414,14 @@ class CacheController(object): # is no date header then we can't do anything about expiring # the cache. elif "date" in response_headers: - date = calendar.timegm(parsedate_tz(response_headers["date"])) + time_tuple = parsedate_tz(response_headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) # cache when there is a max-age > 0 - if "max-age" in cc and cc["max-age"] > 0: + max_age = cc.get("max-age") + if max_age is not None and max_age > 0: logger.debug("Caching b/c date exists and max-age > 0") - expires_time = cc["max-age"] + expires_time = max_age self._cache_set( cache_url, request, @@ -391,7 +436,7 @@ class CacheController(object): if response_headers["expires"]: expires = parsedate_tz(response_headers["expires"]) if expires is not None: - expires_time = calendar.timegm(expires) - date + expires_time = calendar.timegm(expires[:6]) - date else: expires_time = None @@ -408,13 +453,16 @@ class CacheController(object): expires_time, ) - def update_cached_response(self, request, response): + def update_cached_response( + self, request: "PreparedRequest", response: "HTTPResponse" + ) -> "HTTPResponse": """On a 304 we will get a new set of headers that we want to update our cached value with, assuming we have one. This should only ever be called when we've sent an ETag and gotten a 304 as the response. """ + assert request.url is not None cache_url = self.cache_url(request.url) cached_response = self._load_from_cache(request) @@ -434,7 +482,7 @@ class CacheController(object): cached_response.headers.update( dict( (k, v) - for k, v in response.headers.items() + for k, v in response.headers.items() # type: ignore[no-untyped-call] if k.lower() not in excluded_headers ) ) diff --git a/src/pip/_vendor/cachecontrol/filewrapper.py b/src/pip/_vendor/cachecontrol/filewrapper.py index f5ed5f6f6..472ba6001 100644 --- a/src/pip/_vendor/cachecontrol/filewrapper.py +++ b/src/pip/_vendor/cachecontrol/filewrapper.py @@ -2,8 +2,12 @@ # # SPDX-License-Identifier: Apache-2.0 -from tempfile import NamedTemporaryFile import mmap +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING, Any, Callable, Optional + +if TYPE_CHECKING: + from http.client import HTTPResponse class CallbackFileWrapper(object): @@ -25,12 +29,14 @@ class CallbackFileWrapper(object): performance impact. """ - def __init__(self, fp, callback): + def __init__( + self, fp: "HTTPResponse", callback: Optional[Callable[[bytes], None]] + ) -> None: self.__buf = NamedTemporaryFile("rb+", delete=True) self.__fp = fp self.__callback = callback - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: # The vaguaries of garbage collection means that self.__fp is # not always set. By using __getattribute__ and the private # name[0] allows looking up the attribute value and raising an @@ -42,7 +48,7 @@ class CallbackFileWrapper(object): fp = self.__getattribute__("_CallbackFileWrapper__fp") return getattr(fp, name) - def __is_fp_closed(self): + def __is_fp_closed(self) -> bool: try: return self.__fp.fp is None @@ -50,7 +56,8 @@ class CallbackFileWrapper(object): pass try: - return self.__fp.closed + closed: bool = self.__fp.closed + return closed except AttributeError: pass @@ -59,7 +66,7 @@ class CallbackFileWrapper(object): # TODO: Add some logging here... return False - def _close(self): + def _close(self) -> None: if self.__callback: if self.__buf.tell() == 0: # Empty file: @@ -86,8 +93,8 @@ class CallbackFileWrapper(object): # Important when caching big files. self.__buf.close() - def read(self, amt=None): - data = self.__fp.read(amt) + def read(self, amt: Optional[int] = None) -> bytes: + data: bytes = self.__fp.read(amt) if data: # We may be dealing with b'', a sign that things are over: # it's passed e.g. after we've already closed self.__buf. @@ -97,8 +104,8 @@ class CallbackFileWrapper(object): return data - def _safe_read(self, amt): - data = self.__fp._safe_read(amt) + def _safe_read(self, amt: int) -> bytes: + data: bytes = self.__fp._safe_read(amt) # type: ignore[attr-defined] if amt == 2 and data == b"\r\n": # urllib executes this read to toss the CRLF at the end # of the chunk. diff --git a/src/pip/_vendor/cachecontrol/heuristics.py b/src/pip/_vendor/cachecontrol/heuristics.py index ebe4a96f5..1e88ada68 100644 --- a/src/pip/_vendor/cachecontrol/heuristics.py +++ b/src/pip/_vendor/cachecontrol/heuristics.py @@ -4,26 +4,27 @@ import calendar import time - +from datetime import datetime, timedelta, timezone from email.utils import formatdate, parsedate, parsedate_tz +from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional -from datetime import datetime, timedelta +if TYPE_CHECKING: + from pip._vendor.urllib3 import HTTPResponse TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT" -def expire_after(delta, date=None): - date = date or datetime.utcnow() +def expire_after(delta: timedelta, date: Optional[datetime] = None) -> datetime: + date = date or datetime.now(timezone.utc) return date + delta -def datetime_to_header(dt): +def datetime_to_header(dt: datetime) -> str: return formatdate(calendar.timegm(dt.timetuple())) class BaseHeuristic(object): - - def warning(self, response): + def warning(self, response: "HTTPResponse") -> Optional[str]: """ Return a valid 1xx warning header value describing the cache adjustments. @@ -34,7 +35,7 @@ class BaseHeuristic(object): """ return '110 - "Response is Stale"' - def update_headers(self, response): + def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: """Update the response headers with any new headers. NOTE: This SHOULD always include some Warning header to @@ -43,7 +44,7 @@ class BaseHeuristic(object): """ return {} - def apply(self, response): + def apply(self, response: "HTTPResponse") -> "HTTPResponse": updated_headers = self.update_headers(response) if updated_headers: @@ -61,12 +62,12 @@ class OneDayCache(BaseHeuristic): future. """ - def update_headers(self, response): + def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: headers = {} if "expires" not in response.headers: date = parsedate(response.headers["date"]) - expires = expire_after(timedelta(days=1), date=datetime(*date[:6])) + expires = expire_after(timedelta(days=1), date=datetime(*date[:6], tzinfo=timezone.utc)) # type: ignore[misc] headers["expires"] = datetime_to_header(expires) headers["cache-control"] = "public" return headers @@ -77,14 +78,14 @@ class ExpiresAfter(BaseHeuristic): Cache **all** requests for a defined time period. """ - def __init__(self, **kw): + def __init__(self, **kw: Any) -> None: self.delta = timedelta(**kw) - def update_headers(self, response): + def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: expires = expire_after(self.delta) return {"expires": datetime_to_header(expires), "cache-control": "public"} - def warning(self, response): + def warning(self, response: "HTTPResponse") -> Optional[str]: tmpl = "110 - Automatically cached for %s. Response might be stale" return tmpl % self.delta @@ -101,12 +102,23 @@ class LastModified(BaseHeuristic): http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397 Unlike mozilla we limit this to 24-hr. """ + cacheable_by_default_statuses = { - 200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501 + 200, + 203, + 204, + 206, + 300, + 301, + 404, + 405, + 410, + 414, + 501, } - def update_headers(self, resp): - headers = resp.headers + def update_headers(self, resp: "HTTPResponse") -> Dict[str, str]: + headers: Mapping[str, str] = resp.headers if "expires" in headers: return {} @@ -120,9 +132,11 @@ class LastModified(BaseHeuristic): if "date" not in headers or "last-modified" not in headers: return {} - date = calendar.timegm(parsedate_tz(headers["date"])) + time_tuple = parsedate_tz(headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) last_modified = parsedate(headers["last-modified"]) - if date is None or last_modified is None: + if last_modified is None: return {} now = time.time() @@ -135,5 +149,5 @@ class LastModified(BaseHeuristic): expires = date + freshness_lifetime return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))} - def warning(self, resp): + def warning(self, resp: "HTTPResponse") -> Optional[str]: return None diff --git a/src/pip/_vendor/cachecontrol/py.typed b/src/pip/_vendor/cachecontrol/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_vendor/cachecontrol/serialize.py b/src/pip/_vendor/cachecontrol/serialize.py index 7fe1a3e33..f21eaea6f 100644 --- a/src/pip/_vendor/cachecontrol/serialize.py +++ b/src/pip/_vendor/cachecontrol/serialize.py @@ -5,19 +5,23 @@ import base64 import io import json +import pickle import zlib +from typing import IO, TYPE_CHECKING, Any, Mapping, Optional from pip._vendor import msgpack from pip._vendor.requests.structures import CaseInsensitiveDict +from pip._vendor.urllib3 import HTTPResponse -from .compat import HTTPResponse, pickle, text_type +if TYPE_CHECKING: + from pip._vendor.requests import PreparedRequest, Request -def _b64_decode_bytes(b): +def _b64_decode_bytes(b: str) -> bytes: return base64.b64decode(b.encode("ascii")) -def _b64_decode_str(s): +def _b64_decode_str(s: str) -> str: return _b64_decode_bytes(s).decode("utf8") @@ -25,54 +29,57 @@ _default_body_read = object() class Serializer(object): - def dumps(self, request, response, body=None): - response_headers = CaseInsensitiveDict(response.headers) + def dumps( + self, + request: "PreparedRequest", + response: HTTPResponse, + body: Optional[bytes] = None, + ) -> bytes: + response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + response.headers + ) if body is None: # When a body isn't passed in, we'll read the response. We # also update the response with a new file handler to be # sure it acts as though it was never read. body = response.read(decode_content=False) - response._fp = io.BytesIO(body) + response._fp = io.BytesIO(body) # type: ignore[attr-defined] + response.length_remaining = len(body) - # NOTE: This is all a bit weird, but it's really important that on - # Python 2.x these objects are unicode and not str, even when - # they contain only ascii. The problem here is that msgpack - # understands the difference between unicode and bytes and we - # have it set to differentiate between them, however Python 2 - # doesn't know the difference. Forcing these to unicode will be - # enough to have msgpack know the difference. data = { - u"response": { - u"body": body, # Empty bytestring if body is stored separately - u"headers": dict( - (text_type(k), text_type(v)) for k, v in response.headers.items() - ), - u"status": response.status, - u"version": response.version, - u"reason": text_type(response.reason), - u"strict": response.strict, - u"decode_content": response.decode_content, + "response": { + "body": body, # Empty bytestring if body is stored separately + "headers": dict((str(k), str(v)) for k, v in response.headers.items()), # type: ignore[no-untyped-call] + "status": response.status, + "version": response.version, + "reason": str(response.reason), + "decode_content": response.decode_content, } } # Construct our vary headers - data[u"vary"] = {} - if u"vary" in response_headers: - varied_headers = response_headers[u"vary"].split(",") + data["vary"] = {} + if "vary" in response_headers: + varied_headers = response_headers["vary"].split(",") for header in varied_headers: - header = text_type(header).strip() + header = str(header).strip() header_value = request.headers.get(header, None) if header_value is not None: - header_value = text_type(header_value) - data[u"vary"][header] = header_value + header_value = str(header_value) + data["vary"][header] = header_value return b",".join([b"cc=4", msgpack.dumps(data, use_bin_type=True)]) - def loads(self, request, data, body_file=None): + def loads( + self, + request: "PreparedRequest", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: # Short circuit if we've been given an empty set of data if not data: - return + return None # Determine what version of the serializer the data was serialized # with @@ -88,18 +95,23 @@ class Serializer(object): ver = b"cc=0" # Get the version number out of the cc=N - ver = ver.split(b"=", 1)[-1].decode("ascii") + verstr = ver.split(b"=", 1)[-1].decode("ascii") # Dispatch to the actual load method for the given version try: - return getattr(self, "_loads_v{}".format(ver))(request, data, body_file) + return getattr(self, "_loads_v{}".format(verstr))(request, data, body_file) # type: ignore[no-any-return] except AttributeError: # This is a version we don't have a loads function for, so we'll # just treat it as a miss and return None - return + return None - def prepare_response(self, request, cached, body_file=None): + def prepare_response( + self, + request: "Request", + cached: Mapping[str, Any], + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: """Verify our vary headers match and construct a real urllib3 HTTPResponse object. """ @@ -108,23 +120,26 @@ class Serializer(object): # This case is also handled in the controller code when creating # a cache entry, but is left here for backwards compatibility. if "*" in cached.get("vary", {}): - return + return None # Ensure that the Vary headers for the cached response match our # request for header, value in cached.get("vary", {}).items(): if request.headers.get(header, None) != value: - return + return None body_raw = cached["response"].pop("body") - headers = CaseInsensitiveDict(data=cached["response"]["headers"]) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + data=cached["response"]["headers"] + ) if headers.get("transfer-encoding", "") == "chunked": headers.pop("transfer-encoding") cached["response"]["headers"] = headers try: + body: "IO[bytes]" if body_file is None: body = io.BytesIO(body_raw) else: @@ -138,28 +153,46 @@ class Serializer(object): # TypeError: 'str' does not support the buffer interface body = io.BytesIO(body_raw.encode("utf8")) + # Discard any `strict` parameter serialized by older version of cachecontrol. + cached["response"].pop("strict", None) + return HTTPResponse(body=body, preload_content=False, **cached["response"]) - def _loads_v0(self, request, data, body_file=None): + def _loads_v0( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> None: # The original legacy cache data. This doesn't contain enough # information to construct everything we need, so we'll treat this as # a miss. return - def _loads_v1(self, request, data, body_file=None): + def _loads_v1( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: try: cached = pickle.loads(data) except ValueError: - return + return None return self.prepare_response(request, cached, body_file) - def _loads_v2(self, request, data, body_file=None): + def _loads_v2( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: assert body_file is None try: cached = json.loads(zlib.decompress(data).decode("utf8")) except (ValueError, zlib.error): - return + return None # We need to decode the items that we've base64 encoded cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"]) @@ -175,16 +208,26 @@ class Serializer(object): return self.prepare_response(request, cached, body_file) - def _loads_v3(self, request, data, body_file): + def _loads_v3( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> None: # Due to Python 2 encoding issues, it's impossible to know for sure # exactly how to load v3 entries, thus we'll treat these as a miss so # that they get rewritten out as v4 entries. return - def _loads_v4(self, request, data, body_file=None): + def _loads_v4( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: try: cached = msgpack.loads(data, raw=False) except ValueError: - return + return None return self.prepare_response(request, cached, body_file) diff --git a/src/pip/_vendor/cachecontrol/wrapper.py b/src/pip/_vendor/cachecontrol/wrapper.py index b6ee7f203..293e69fe7 100644 --- a/src/pip/_vendor/cachecontrol/wrapper.py +++ b/src/pip/_vendor/cachecontrol/wrapper.py @@ -2,21 +2,30 @@ # # SPDX-License-Identifier: Apache-2.0 -from .adapter import CacheControlAdapter -from .cache import DictCache +from typing import TYPE_CHECKING, Collection, Optional, Type + +from pip._vendor.cachecontrol.adapter import CacheControlAdapter +from pip._vendor.cachecontrol.cache import DictCache + +if TYPE_CHECKING: + from pip._vendor import requests + + from pip._vendor.cachecontrol.cache import BaseCache + from pip._vendor.cachecontrol.controller import CacheController + from pip._vendor.cachecontrol.heuristics import BaseHeuristic + from pip._vendor.cachecontrol.serialize import Serializer def CacheControl( - sess, - cache=None, - cache_etags=True, - serializer=None, - heuristic=None, - controller_class=None, - adapter_class=None, - cacheable_methods=None, -): - + sess: "requests.Session", + cache: Optional["BaseCache"] = None, + cache_etags: bool = True, + serializer: Optional["Serializer"] = None, + heuristic: Optional["BaseHeuristic"] = None, + controller_class: Optional[Type["CacheController"]] = None, + adapter_class: Optional[Type[CacheControlAdapter]] = None, + cacheable_methods: Optional[Collection[str]] = None, +) -> "requests.Session": cache = DictCache() if cache is None else cache adapter_class = adapter_class or CacheControlAdapter adapter = adapter_class( diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index dcf89dc04..d0f4c71cc 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -1,4 +1,4 @@ -CacheControl==0.12.11 # Make sure to update the license in pyproject.toml for this. +CacheControl==0.13.0 # Make sure to update the license in pyproject.toml for this. colorama==0.4.6 distlib==0.3.6 distro==1.8.0 From 9fb93c478ef7d5e1423cc66467bb63c686864828 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Jun 2023 14:00:15 -0400 Subject: [PATCH 024/156] mypy fix. --- src/pip/_internal/network/cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index d6b8ccdcf..a4d136205 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -3,7 +3,8 @@ import os from contextlib import contextmanager -from typing import BinaryIO, Generator, Optional +from datetime import datetime +from typing import BinaryIO, Generator, Optional, Union from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache from pip._vendor.cachecontrol.caches import SeparateBodyFileCache @@ -62,7 +63,9 @@ class SafeFileCache(SeparateBodyBaseCache): replace(f.name, path) - def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None: + def set( + self, key: str, value: bytes, expires: Union[int, datetime, None] = None + ) -> None: path = self._get_cache_path(key) self._write(path, value) From 28590a0a0809b3bb8999b4d08aa93bd9ffb3458d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 09:29:35 -0400 Subject: [PATCH 025/156] Improve documentation of caching and the cache subcommand. --- docs/html/topics/caching.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/html/topics/caching.md b/docs/html/topics/caching.md index 954cebe40..19bd064a7 100644 --- a/docs/html/topics/caching.md +++ b/docs/html/topics/caching.md @@ -27,6 +27,12 @@ While this cache attempts to minimize network activity, it does not prevent network access altogether. If you want a local install solution that circumvents accessing PyPI, see {ref}`Installing from local packages`. +In versions prior to 23.2, this cache was stored in a directory called `http` in +the main cache directory (see below for its location). In 23.2 and later, a new +cache format is used, stored in a directory called `http-v2`. If you have +completely switched to newer versions of `pip`, you may wish to delete the old +directory. + (wheel-caching)= ### Locally built wheels @@ -124,11 +130,11 @@ The {ref}`pip cache` command can be used to manage pip's cache. ### Removing a single package -`pip cache remove setuptools` removes all wheel files related to setuptools from pip's cache. +`pip cache remove setuptools` removes all wheel files related to setuptools from pip's cache. HTTP cache files are not removed at this time. ### Removing the cache -`pip cache purge` will clear all wheel files from pip's cache. +`pip cache purge` will clear all files from pip's wheel and HTTP caches. ### Listing cached files From dcd2d5e344f27149789f05edb9da45994eac2473 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 09:30:30 -0400 Subject: [PATCH 026/156] Update CacheControl to 0.13.1. --- src/pip/_vendor/cachecontrol/__init__.py | 2 +- src/pip/_vendor/cachecontrol/_cmd.py | 5 +- src/pip/_vendor/cachecontrol/adapter.py | 51 ++++---- src/pip/_vendor/cachecontrol/cache.py | 18 +-- .../_vendor/cachecontrol/caches/file_cache.py | 17 +-- .../cachecontrol/caches/redis_cache.py | 10 +- src/pip/_vendor/cachecontrol/controller.py | 58 +++++---- src/pip/_vendor/cachecontrol/filewrapper.py | 9 +- src/pip/_vendor/cachecontrol/heuristics.py | 23 ++-- src/pip/_vendor/cachecontrol/serialize.py | 111 +++++++----------- src/pip/_vendor/cachecontrol/wrapper.py | 19 +-- src/pip/_vendor/vendor.txt | 2 +- 12 files changed, 149 insertions(+), 176 deletions(-) diff --git a/src/pip/_vendor/cachecontrol/__init__.py b/src/pip/_vendor/cachecontrol/__init__.py index 3701cdd6b..4d20bc9b1 100644 --- a/src/pip/_vendor/cachecontrol/__init__.py +++ b/src/pip/_vendor/cachecontrol/__init__.py @@ -8,7 +8,7 @@ Make it easy to import from cachecontrol without long namespaces. """ __author__ = "Eric Larson" __email__ = "eric@ionrock.org" -__version__ = "0.13.0" +__version__ = "0.13.1" from pip._vendor.cachecontrol.adapter import CacheControlAdapter from pip._vendor.cachecontrol.controller import CacheController diff --git a/src/pip/_vendor/cachecontrol/_cmd.py b/src/pip/_vendor/cachecontrol/_cmd.py index ab4dac3dd..2c84208a5 100644 --- a/src/pip/_vendor/cachecontrol/_cmd.py +++ b/src/pip/_vendor/cachecontrol/_cmd.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import logging from argparse import ArgumentParser @@ -36,7 +37,7 @@ def get_session() -> requests.Session: return sess -def get_args() -> "Namespace": +def get_args() -> Namespace: parser = ArgumentParser() parser.add_argument("url", help="The URL to try and cache") return parser.parse_args() @@ -53,7 +54,7 @@ def main() -> None: setup_logging() # try setting the cache - cache_controller: "CacheController" = ( + cache_controller: CacheController = ( sess.cache_controller # type: ignore[attr-defined] ) cache_controller.cache_response(resp.request, resp.raw) diff --git a/src/pip/_vendor/cachecontrol/adapter.py b/src/pip/_vendor/cachecontrol/adapter.py index 83c08e003..3e83e308d 100644 --- a/src/pip/_vendor/cachecontrol/adapter.py +++ b/src/pip/_vendor/cachecontrol/adapter.py @@ -1,11 +1,12 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import functools import types import zlib -from typing import TYPE_CHECKING, Any, Collection, Mapping, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Collection, Mapping from pip._vendor.requests.adapters import HTTPAdapter @@ -27,16 +28,16 @@ class CacheControlAdapter(HTTPAdapter): def __init__( self, - cache: Optional["BaseCache"] = None, + cache: BaseCache | None = None, cache_etags: bool = True, - controller_class: Optional[Type[CacheController]] = None, - serializer: Optional["Serializer"] = None, - heuristic: Optional["BaseHeuristic"] = None, - cacheable_methods: Optional[Collection[str]] = None, + controller_class: type[CacheController] | None = None, + serializer: Serializer | None = None, + heuristic: BaseHeuristic | None = None, + cacheable_methods: Collection[str] | None = None, *args: Any, **kw: Any, ) -> None: - super(CacheControlAdapter, self).__init__(*args, **kw) + super().__init__(*args, **kw) self.cache = DictCache() if cache is None else cache self.heuristic = heuristic self.cacheable_methods = cacheable_methods or ("GET",) @@ -48,16 +49,14 @@ class CacheControlAdapter(HTTPAdapter): def send( self, - request: "PreparedRequest", + request: PreparedRequest, stream: bool = False, - timeout: Union[None, float, Tuple[float, float], Tuple[float, None]] = None, - verify: Union[bool, str] = True, - cert: Union[ - None, bytes, str, Tuple[Union[bytes, str], Union[bytes, str]] - ] = None, - proxies: Optional[Mapping[str, str]] = None, - cacheable_methods: Optional[Collection[str]] = None, - ) -> "Response": + timeout: None | float | tuple[float, float] | tuple[float, None] = None, + verify: bool | str = True, + cert: (None | bytes | str | tuple[bytes | str, bytes | str]) = None, + proxies: Mapping[str, str] | None = None, + cacheable_methods: Collection[str] | None = None, + ) -> Response: """ Send a request. Use the request information to see if it exists in the cache and cache the response if we need to and can. @@ -74,19 +73,17 @@ class CacheControlAdapter(HTTPAdapter): # check for etags and add headers if appropriate request.headers.update(self.controller.conditional_headers(request)) - resp = super(CacheControlAdapter, self).send( - request, stream, timeout, verify, cert, proxies - ) + resp = super().send(request, stream, timeout, verify, cert, proxies) return resp def build_response( self, - request: "PreparedRequest", - response: "HTTPResponse", + request: PreparedRequest, + response: HTTPResponse, from_cache: bool = False, - cacheable_methods: Optional[Collection[str]] = None, - ) -> "Response": + cacheable_methods: Collection[str] | None = None, + ) -> Response: """ Build a response by making a request or using the cache. @@ -137,7 +134,7 @@ class CacheControlAdapter(HTTPAdapter): if response.chunked: super_update_chunk_length = response._update_chunk_length # type: ignore[attr-defined] - def _update_chunk_length(self: "HTTPResponse") -> None: + def _update_chunk_length(self: HTTPResponse) -> None: super_update_chunk_length() if self.chunk_left == 0: self._fp._close() # type: ignore[attr-defined] @@ -146,9 +143,7 @@ class CacheControlAdapter(HTTPAdapter): _update_chunk_length, response ) - resp: "Response" = super( # type: ignore[no-untyped-call] - CacheControlAdapter, self - ).build_response(request, response) + resp: Response = super().build_response(request, response) # type: ignore[no-untyped-call] # See if we should invalidate the cache. if request.method in self.invalidating_methods and resp.ok: @@ -163,4 +158,4 @@ class CacheControlAdapter(HTTPAdapter): def close(self) -> None: self.cache.close() - super(CacheControlAdapter, self).close() # type: ignore[no-untyped-call] + super().close() # type: ignore[no-untyped-call] diff --git a/src/pip/_vendor/cachecontrol/cache.py b/src/pip/_vendor/cachecontrol/cache.py index 61031d234..3293b0057 100644 --- a/src/pip/_vendor/cachecontrol/cache.py +++ b/src/pip/_vendor/cachecontrol/cache.py @@ -6,19 +6,21 @@ The cache object API for implementing caches. The default is a thread safe in-memory dictionary. """ +from __future__ import annotations + from threading import Lock -from typing import IO, TYPE_CHECKING, MutableMapping, Optional, Union +from typing import IO, TYPE_CHECKING, MutableMapping if TYPE_CHECKING: from datetime import datetime -class BaseCache(object): - def get(self, key: str) -> Optional[bytes]: +class BaseCache: + def get(self, key: str) -> bytes | None: raise NotImplementedError() def set( - self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + self, key: str, value: bytes, expires: int | datetime | None = None ) -> None: raise NotImplementedError() @@ -30,15 +32,15 @@ class BaseCache(object): class DictCache(BaseCache): - def __init__(self, init_dict: Optional[MutableMapping[str, bytes]] = None) -> None: + def __init__(self, init_dict: MutableMapping[str, bytes] | None = None) -> None: self.lock = Lock() self.data = init_dict or {} - def get(self, key: str) -> Optional[bytes]: + def get(self, key: str) -> bytes | None: return self.data.get(key, None) def set( - self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + self, key: str, value: bytes, expires: int | datetime | None = None ) -> None: with self.lock: self.data.update({key: value}) @@ -65,7 +67,7 @@ class SeparateBodyBaseCache(BaseCache): def set_body(self, key: str, body: bytes) -> None: raise NotImplementedError() - def get_body(self, key: str) -> Optional["IO[bytes]"]: + def get_body(self, key: str) -> IO[bytes] | None: """ Return the body as file-like object. """ diff --git a/src/pip/_vendor/cachecontrol/caches/file_cache.py b/src/pip/_vendor/cachecontrol/caches/file_cache.py index 0437c4e8a..1fd280130 100644 --- a/src/pip/_vendor/cachecontrol/caches/file_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/file_cache.py @@ -1,11 +1,12 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import hashlib import os from textwrap import dedent -from typing import IO, TYPE_CHECKING, Optional, Type, Union +from typing import IO, TYPE_CHECKING from pip._vendor.cachecontrol.cache import BaseCache, SeparateBodyBaseCache from pip._vendor.cachecontrol.controller import CacheController @@ -16,7 +17,7 @@ if TYPE_CHECKING: from filelock import BaseFileLock -def _secure_open_write(filename: str, fmode: int) -> "IO[bytes]": +def _secure_open_write(filename: str, fmode: int) -> IO[bytes]: # We only want to write to this file, so open it in write only mode flags = os.O_WRONLY @@ -39,7 +40,7 @@ def _secure_open_write(filename: str, fmode: int) -> "IO[bytes]": # there try: os.remove(filename) - except (IOError, OSError): + except OSError: # The file must not exist already, so we can just skip ahead to opening pass @@ -66,7 +67,7 @@ class _FileCacheMixin: forever: bool = False, filemode: int = 0o0600, dirmode: int = 0o0700, - lock_class: Optional[Type["BaseFileLock"]] = None, + lock_class: type[BaseFileLock] | None = None, ) -> None: try: if lock_class is None: @@ -100,7 +101,7 @@ class _FileCacheMixin: parts = list(hashed[:5]) + [hashed] return os.path.join(self.directory, *parts) - def get(self, key: str) -> Optional[bytes]: + def get(self, key: str) -> bytes | None: name = self._fn(key) try: with open(name, "rb") as fh: @@ -110,7 +111,7 @@ class _FileCacheMixin: return None def set( - self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + self, key: str, value: bytes, expires: int | datetime | None = None ) -> None: name = self._fn(key) self._write(name, value) @@ -122,7 +123,7 @@ class _FileCacheMixin: # Make sure the directory exists try: os.makedirs(os.path.dirname(path), self.dirmode) - except (IOError, OSError): + except OSError: pass with self.lock_class(path + ".lock"): @@ -155,7 +156,7 @@ class SeparateBodyFileCache(_FileCacheMixin, SeparateBodyBaseCache): peak memory usage. """ - def get_body(self, key: str) -> Optional["IO[bytes]"]: + def get_body(self, key: str) -> IO[bytes] | None: name = self._fn(key) + ".body" try: return open(name, "rb") diff --git a/src/pip/_vendor/cachecontrol/caches/redis_cache.py b/src/pip/_vendor/cachecontrol/caches/redis_cache.py index f7ae45d38..f4f68c47b 100644 --- a/src/pip/_vendor/cachecontrol/caches/redis_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/redis_cache.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -from __future__ import division from datetime import datetime, timezone -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from pip._vendor.cachecontrol.cache import BaseCache @@ -14,14 +14,14 @@ if TYPE_CHECKING: class RedisCache(BaseCache): - def __init__(self, conn: "Redis[bytes]") -> None: + def __init__(self, conn: Redis[bytes]) -> None: self.conn = conn - def get(self, key: str) -> Optional[bytes]: + def get(self, key: str) -> bytes | None: return self.conn.get(key) def set( - self, key: str, value: bytes, expires: Optional[Union[int, datetime]] = None + self, key: str, value: bytes, expires: int | datetime | None = None ) -> None: if not expires: self.conn.set(key, value) diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 3365d9621..586b9f97b 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -5,12 +5,14 @@ """ The httplib2 algorithms ported for use with requests. """ +from __future__ import annotations + import calendar import logging import re import time from email.utils import parsedate_tz -from typing import TYPE_CHECKING, Collection, Dict, Mapping, Optional, Tuple, Union +from typing import TYPE_CHECKING, Collection, Mapping from pip._vendor.requests.structures import CaseInsensitiveDict @@ -32,7 +34,7 @@ URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") PERMANENT_REDIRECT_STATUSES = (301, 308) -def parse_uri(uri: str) -> Tuple[str, str, str, str, str]: +def parse_uri(uri: str) -> tuple[str, str, str, str, str]: """Parses a URI using the regex given in Appendix B of RFC 3986. (scheme, authority, path, query, fragment) = parse_uri(uri) @@ -43,15 +45,15 @@ def parse_uri(uri: str) -> Tuple[str, str, str, str, str]: return (groups[1], groups[3], groups[4], groups[6], groups[8]) -class CacheController(object): +class CacheController: """An interface to see if request should cached or not.""" def __init__( self, - cache: Optional["BaseCache"] = None, + cache: BaseCache | None = None, cache_etags: bool = True, - serializer: Optional[Serializer] = None, - status_codes: Optional[Collection[int]] = None, + serializer: Serializer | None = None, + status_codes: Collection[int] | None = None, ): self.cache = DictCache() if cache is None else cache self.cache_etags = cache_etags @@ -82,9 +84,7 @@ class CacheController(object): def cache_url(cls, uri: str) -> str: return cls._urlnorm(uri) - def parse_cache_control( - self, headers: Mapping[str, str] - ) -> Dict[str, Optional[int]]: + def parse_cache_control(self, headers: Mapping[str, str]) -> dict[str, int | None]: known_directives = { # https://tools.ietf.org/html/rfc7234#section-5.2 "max-age": (int, True), @@ -103,7 +103,7 @@ class CacheController(object): cc_headers = headers.get("cache-control", headers.get("Cache-Control", "")) - retval: Dict[str, Optional[int]] = {} + retval: dict[str, int | None] = {} for cc_directive in cc_headers.split(","): if not cc_directive.strip(): @@ -138,7 +138,7 @@ class CacheController(object): return retval - def _load_from_cache(self, request: "PreparedRequest") -> Optional["HTTPResponse"]: + def _load_from_cache(self, request: PreparedRequest) -> HTTPResponse | None: """ Load a cached response, or return None if it's not available. """ @@ -159,9 +159,7 @@ class CacheController(object): logger.warning("Cache entry deserialization failed, entry ignored") return result - def cached_request( - self, request: "PreparedRequest" - ) -> Union["HTTPResponse", "Literal[False]"]: + def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[False]: """ Return a cached response if it exists in the cache, otherwise return False. @@ -271,7 +269,7 @@ class CacheController(object): # return the original handler return False - def conditional_headers(self, request: "PreparedRequest") -> Dict[str, str]: + def conditional_headers(self, request: PreparedRequest) -> dict[str, str]: resp = self._load_from_cache(request) new_headers = {} @@ -289,10 +287,10 @@ class CacheController(object): def _cache_set( self, cache_url: str, - request: "PreparedRequest", - response: "HTTPResponse", - body: Optional[bytes] = None, - expires_time: Optional[int] = None, + request: PreparedRequest, + response: HTTPResponse, + body: bytes | None = None, + expires_time: int | None = None, ) -> None: """ Store the data in the cache. @@ -318,10 +316,10 @@ class CacheController(object): def cache_response( self, - request: "PreparedRequest", - response: "HTTPResponse", - body: Optional[bytes] = None, - status_codes: Optional[Collection[int]] = None, + request: PreparedRequest, + response: HTTPResponse, + body: bytes | None = None, + status_codes: Collection[int] | None = None, ) -> None: """ Algorithm for caching requests. @@ -400,7 +398,7 @@ class CacheController(object): expires_time = max(expires_time, 14 * 86400) - logger.debug("etag object cached for {0} seconds".format(expires_time)) + logger.debug(f"etag object cached for {expires_time} seconds") logger.debug("Caching due to etag") self._cache_set(cache_url, request, response, body, expires_time) @@ -441,7 +439,7 @@ class CacheController(object): expires_time = None logger.debug( - "Caching b/c of expires header. expires in {0} seconds".format( + "Caching b/c of expires header. expires in {} seconds".format( expires_time ) ) @@ -454,8 +452,8 @@ class CacheController(object): ) def update_cached_response( - self, request: "PreparedRequest", response: "HTTPResponse" - ) -> "HTTPResponse": + self, request: PreparedRequest, response: HTTPResponse + ) -> HTTPResponse: """On a 304 we will get a new set of headers that we want to update our cached value with, assuming we have one. @@ -480,11 +478,11 @@ class CacheController(object): excluded_headers = ["content-length"] cached_response.headers.update( - dict( - (k, v) + { + k: v for k, v in response.headers.items() # type: ignore[no-untyped-call] if k.lower() not in excluded_headers - ) + } ) # we want a 200 b/c we have content via the cache diff --git a/src/pip/_vendor/cachecontrol/filewrapper.py b/src/pip/_vendor/cachecontrol/filewrapper.py index 472ba6001..25143902a 100644 --- a/src/pip/_vendor/cachecontrol/filewrapper.py +++ b/src/pip/_vendor/cachecontrol/filewrapper.py @@ -1,16 +1,17 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import mmap from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Any, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable if TYPE_CHECKING: from http.client import HTTPResponse -class CallbackFileWrapper(object): +class CallbackFileWrapper: """ Small wrapper around a fp object which will tee everything read into a buffer, and when that file is closed it will execute a callback with the @@ -30,7 +31,7 @@ class CallbackFileWrapper(object): """ def __init__( - self, fp: "HTTPResponse", callback: Optional[Callable[[bytes], None]] + self, fp: HTTPResponse, callback: Callable[[bytes], None] | None ) -> None: self.__buf = NamedTemporaryFile("rb+", delete=True) self.__fp = fp @@ -93,7 +94,7 @@ class CallbackFileWrapper(object): # Important when caching big files. self.__buf.close() - def read(self, amt: Optional[int] = None) -> bytes: + def read(self, amt: int | None = None) -> bytes: data: bytes = self.__fp.read(amt) if data: # We may be dealing with b'', a sign that things are over: diff --git a/src/pip/_vendor/cachecontrol/heuristics.py b/src/pip/_vendor/cachecontrol/heuristics.py index 1e88ada68..b9d72ca4a 100644 --- a/src/pip/_vendor/cachecontrol/heuristics.py +++ b/src/pip/_vendor/cachecontrol/heuristics.py @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import calendar import time from datetime import datetime, timedelta, timezone from email.utils import formatdate, parsedate, parsedate_tz -from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional +from typing import TYPE_CHECKING, Any, Mapping if TYPE_CHECKING: from pip._vendor.urllib3 import HTTPResponse @@ -14,7 +15,7 @@ if TYPE_CHECKING: TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT" -def expire_after(delta: timedelta, date: Optional[datetime] = None) -> datetime: +def expire_after(delta: timedelta, date: datetime | None = None) -> datetime: date = date or datetime.now(timezone.utc) return date + delta @@ -23,8 +24,8 @@ def datetime_to_header(dt: datetime) -> str: return formatdate(calendar.timegm(dt.timetuple())) -class BaseHeuristic(object): - def warning(self, response: "HTTPResponse") -> Optional[str]: +class BaseHeuristic: + def warning(self, response: HTTPResponse) -> str | None: """ Return a valid 1xx warning header value describing the cache adjustments. @@ -35,7 +36,7 @@ class BaseHeuristic(object): """ return '110 - "Response is Stale"' - def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: + def update_headers(self, response: HTTPResponse) -> dict[str, str]: """Update the response headers with any new headers. NOTE: This SHOULD always include some Warning header to @@ -44,7 +45,7 @@ class BaseHeuristic(object): """ return {} - def apply(self, response: "HTTPResponse") -> "HTTPResponse": + def apply(self, response: HTTPResponse) -> HTTPResponse: updated_headers = self.update_headers(response) if updated_headers: @@ -62,7 +63,7 @@ class OneDayCache(BaseHeuristic): future. """ - def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: + def update_headers(self, response: HTTPResponse) -> dict[str, str]: headers = {} if "expires" not in response.headers: @@ -81,11 +82,11 @@ class ExpiresAfter(BaseHeuristic): def __init__(self, **kw: Any) -> None: self.delta = timedelta(**kw) - def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: + def update_headers(self, response: HTTPResponse) -> dict[str, str]: expires = expire_after(self.delta) return {"expires": datetime_to_header(expires), "cache-control": "public"} - def warning(self, response: "HTTPResponse") -> Optional[str]: + def warning(self, response: HTTPResponse) -> str | None: tmpl = "110 - Automatically cached for %s. Response might be stale" return tmpl % self.delta @@ -117,7 +118,7 @@ class LastModified(BaseHeuristic): 501, } - def update_headers(self, resp: "HTTPResponse") -> Dict[str, str]: + def update_headers(self, resp: HTTPResponse) -> dict[str, str]: headers: Mapping[str, str] = resp.headers if "expires" in headers: @@ -149,5 +150,5 @@ class LastModified(BaseHeuristic): expires = date + freshness_lifetime return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))} - def warning(self, resp: "HTTPResponse") -> Optional[str]: + def warning(self, resp: HTTPResponse) -> str | None: return None diff --git a/src/pip/_vendor/cachecontrol/serialize.py b/src/pip/_vendor/cachecontrol/serialize.py index f21eaea6f..f9e967c3c 100644 --- a/src/pip/_vendor/cachecontrol/serialize.py +++ b/src/pip/_vendor/cachecontrol/serialize.py @@ -1,39 +1,27 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -import base64 import io -import json -import pickle -import zlib -from typing import IO, TYPE_CHECKING, Any, Mapping, Optional +from typing import IO, TYPE_CHECKING, Any, Mapping, cast from pip._vendor import msgpack from pip._vendor.requests.structures import CaseInsensitiveDict from pip._vendor.urllib3 import HTTPResponse if TYPE_CHECKING: - from pip._vendor.requests import PreparedRequest, Request + from pip._vendor.requests import PreparedRequest -def _b64_decode_bytes(b: str) -> bytes: - return base64.b64decode(b.encode("ascii")) +class Serializer: + serde_version = "4" - -def _b64_decode_str(s: str) -> str: - return _b64_decode_bytes(s).decode("utf8") - - -_default_body_read = object() - - -class Serializer(object): def dumps( self, - request: "PreparedRequest", + request: PreparedRequest, response: HTTPResponse, - body: Optional[bytes] = None, + body: bytes | None = None, ) -> bytes: response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( response.headers @@ -50,7 +38,7 @@ class Serializer(object): data = { "response": { "body": body, # Empty bytestring if body is stored separately - "headers": dict((str(k), str(v)) for k, v in response.headers.items()), # type: ignore[no-untyped-call] + "headers": {str(k): str(v) for k, v in response.headers.items()}, # type: ignore[no-untyped-call] "status": response.status, "version": response.version, "reason": str(response.reason), @@ -69,14 +57,17 @@ class Serializer(object): header_value = str(header_value) data["vary"][header] = header_value - return b",".join([b"cc=4", msgpack.dumps(data, use_bin_type=True)]) + return b",".join([f"cc={self.serde_version}".encode(), self.serialize(data)]) + + def serialize(self, data: dict[str, Any]) -> bytes: + return cast(bytes, msgpack.dumps(data, use_bin_type=True)) def loads( self, - request: "PreparedRequest", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: # Short circuit if we've been given an empty set of data if not data: return None @@ -99,7 +90,7 @@ class Serializer(object): # Dispatch to the actual load method for the given version try: - return getattr(self, "_loads_v{}".format(verstr))(request, data, body_file) # type: ignore[no-any-return] + return getattr(self, f"_loads_v{verstr}")(request, data, body_file) # type: ignore[no-any-return] except AttributeError: # This is a version we don't have a loads function for, so we'll @@ -108,10 +99,10 @@ class Serializer(object): def prepare_response( self, - request: "Request", + request: PreparedRequest, cached: Mapping[str, Any], - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: """Verify our vary headers match and construct a real urllib3 HTTPResponse object. """ @@ -139,7 +130,7 @@ class Serializer(object): cached["response"]["headers"] = headers try: - body: "IO[bytes]" + body: IO[bytes] if body_file is None: body = io.BytesIO(body_raw) else: @@ -160,71 +151,53 @@ class Serializer(object): def _loads_v0( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, + body_file: IO[bytes] | None = None, ) -> None: # The original legacy cache data. This doesn't contain enough # information to construct everything we need, so we'll treat this as # a miss. - return + return None def _loads_v1( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: - try: - cached = pickle.loads(data) - except ValueError: - return None - - return self.prepare_response(request, cached, body_file) + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: + # The "v1" pickled cache format. This is no longer supported + # for security reasons, so we treat it as a miss. + return None def _loads_v2( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: - assert body_file is None - try: - cached = json.loads(zlib.decompress(data).decode("utf8")) - except (ValueError, zlib.error): - return None - - # We need to decode the items that we've base64 encoded - cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"]) - cached["response"]["headers"] = dict( - (_b64_decode_str(k), _b64_decode_str(v)) - for k, v in cached["response"]["headers"].items() - ) - cached["response"]["reason"] = _b64_decode_str(cached["response"]["reason"]) - cached["vary"] = dict( - (_b64_decode_str(k), _b64_decode_str(v) if v is not None else v) - for k, v in cached["vary"].items() - ) - - return self.prepare_response(request, cached, body_file) + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: + # The "v2" compressed base64 cache format. + # This has been removed due to age and poor size/performance + # characteristics, so we treat it as a miss. + return None def _loads_v3( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, + body_file: IO[bytes] | None = None, ) -> None: # Due to Python 2 encoding issues, it's impossible to know for sure # exactly how to load v3 entries, thus we'll treat these as a miss so # that they get rewritten out as v4 entries. - return + return None def _loads_v4( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: try: cached = msgpack.loads(data, raw=False) except ValueError: diff --git a/src/pip/_vendor/cachecontrol/wrapper.py b/src/pip/_vendor/cachecontrol/wrapper.py index 293e69fe7..f618bc363 100644 --- a/src/pip/_vendor/cachecontrol/wrapper.py +++ b/src/pip/_vendor/cachecontrol/wrapper.py @@ -1,8 +1,9 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -from typing import TYPE_CHECKING, Collection, Optional, Type +from typing import TYPE_CHECKING, Collection from pip._vendor.cachecontrol.adapter import CacheControlAdapter from pip._vendor.cachecontrol.cache import DictCache @@ -17,15 +18,15 @@ if TYPE_CHECKING: def CacheControl( - sess: "requests.Session", - cache: Optional["BaseCache"] = None, + sess: requests.Session, + cache: BaseCache | None = None, cache_etags: bool = True, - serializer: Optional["Serializer"] = None, - heuristic: Optional["BaseHeuristic"] = None, - controller_class: Optional[Type["CacheController"]] = None, - adapter_class: Optional[Type[CacheControlAdapter]] = None, - cacheable_methods: Optional[Collection[str]] = None, -) -> "requests.Session": + serializer: Serializer | None = None, + heuristic: BaseHeuristic | None = None, + controller_class: type[CacheController] | None = None, + adapter_class: type[CacheControlAdapter] | None = None, + cacheable_methods: Collection[str] | None = None, +) -> requests.Session: cache = DictCache() if cache is None else cache adapter_class = adapter_class or CacheControlAdapter adapter = adapter_class( diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index d0f4c71cc..c6809dfd6 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -1,4 +1,4 @@ -CacheControl==0.13.0 # Make sure to update the license in pyproject.toml for this. +CacheControl==0.13.1 # Make sure to update the license in pyproject.toml for this. colorama==0.4.6 distlib==0.3.6 distro==1.8.0 From 8aa17580ed623d926795e0cfb8885b4a4b4e044e Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 13 Jul 2023 14:57:18 +0200 Subject: [PATCH 027/156] dropped unused attribute --- src/pip/_internal/resolution/resolvelib/requirements.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index 31a515da9..e23b948ff 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -52,7 +52,6 @@ class SpecifierRequirement(Requirement): making this a requirement on the base only. """ assert ireq.link is None, "This is a link, not a specifier" - self._drop_extras: bool = drop_extras self._ireq = ireq if not drop_extras else install_req_drop_extras(ireq) self._extras = frozenset(self._ireq.extras) From faa3289a94c59cdf647ce9d9c9277714c9363a62 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 13 Jul 2023 16:40:56 +0200 Subject: [PATCH 028/156] use regex for requirement update --- src/pip/_internal/req/constructors.py | 28 +++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 8b1438afe..ee38b9a61 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -58,6 +58,26 @@ def convert_extras(extras: Optional[str]) -> Set[str]: return get_requirement("placeholder" + extras.lower()).extras +def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requirement: + """ + Returns a new requirement based on the given one, with the supplied extras. If the + given requirement already has extras those are replaced (or dropped if no new extras + are given). + """ + match: re.Match = re.fullmatch(r"([^;\[<>~=]+)(\[[^\]]*\])?(.*)", str(req)) + # ireq.req is a valid requirement so the regex should match + assert match is not None + pre: Optional[str] = match.group(1) + post: Optional[str] = match.group(3) + assert pre is not None and post is not None + extras: str = ( + "[%s]" % ",".join(sorted(new_extras)) + if new_extras + else "" + ) + return Requirement(pre + extras + post) + + def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: """Parses an editable requirement into: - a requirement name @@ -513,10 +533,8 @@ def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement: any extras. Sets the original requirement as the new one's parent (comes_from). """ - req = Requirement(str(ireq.req)) - req.extras = {} return InstallRequirement( - req=req, + req=_set_requirement_extras(ireq.req, set()), comes_from=ireq, editable=ireq.editable, link=ireq.link, @@ -542,8 +560,6 @@ def install_req_extend_extras( Makes a shallow copy of the ireq object. """ result = copy.copy(ireq) - req = Requirement(str(ireq.req)) - req.extras.update(extras) - result.req = req result.extras = {*ireq.extras, *extras} + result.req = _set_requirement_extras(ireq.req, result.extras) return result From 7e8da6176f9da32e44b8a1515e450ca8158a356a Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 13 Jul 2023 17:02:53 +0200 Subject: [PATCH 029/156] clarification --- src/pip/_internal/req/constructors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index ee38b9a61..f97bded98 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -65,7 +65,7 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme are given). """ match: re.Match = re.fullmatch(r"([^;\[<>~=]+)(\[[^\]]*\])?(.*)", str(req)) - # ireq.req is a valid requirement so the regex should match + # ireq.req is a valid requirement so the regex should always match assert match is not None pre: Optional[str] = match.group(1) post: Optional[str] = match.group(3) From ff9aeae0d2e39720e40e8fffae942d659495fd84 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 25 Jul 2023 15:36:33 +0200 Subject: [PATCH 030/156] added resolver test case --- tests/functional/test_new_resolver.py | 82 +++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index fc52ab9c8..88dd635ae 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -2272,6 +2272,88 @@ def test_new_resolver_dont_backtrack_on_extra_if_base_constrained( script.assert_installed(pkg="1.0", dep="1.0") +@pytest.mark.parametrize("swap_order", (True, False)) +@pytest.mark.parametrize("two_extras", (True, False)) +def test_new_resolver_dont_backtrack_on_extra_if_base_constrained_in_requirement( + script: PipTestEnvironment, swap_order: bool, two_extras: bool +) -> None: + """ + Verify that a requirement with a constraint on a package (either on the base + on the base with an extra) causes the resolver to infer the same constraint for + any (other) extras with the same base. + + :param swap_order: swap the order the install specifiers appear in + :param two_extras: also add an extra for the constrained specifier + """ + create_basic_wheel_for_package(script, "dep", "1.0") + create_basic_wheel_for_package( + script, "pkg", "1.0", extras={"ext1": ["dep"], "ext2": ["dep"]} + ) + create_basic_wheel_for_package( + script, "pkg", "2.0", extras={"ext1": ["dep"], "ext2": ["dep"]} + ) + + to_install: tuple[str, str] = ( + "pkg[ext1]", "pkg[ext2]==1.0" if two_extras else "pkg==1.0" + ) + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + *(to_install if not swap_order else reversed(to_install)), + ) + assert "pkg-2.0" not in result.stdout, "Should not try 2.0 due to constraint" + script.assert_installed(pkg="1.0", dep="1.0") + + +@pytest.mark.parametrize("swap_order", (True, False)) +@pytest.mark.parametrize("two_extras", (True, False)) +def test_new_resolver_dont_backtrack_on_conflicting_constraints_on_extras( + script: PipTestEnvironment, swap_order: bool, two_extras: bool +) -> None: + """ + Verify that conflicting constraints on the same package with different + extras cause the resolver to trivially reject the request rather than + trying any candidates. + + :param swap_order: swap the order the install specifiers appear in + :param two_extras: also add an extra for the second specifier + """ + create_basic_wheel_for_package(script, "dep", "1.0") + create_basic_wheel_for_package( + script, "pkg", "1.0", extras={"ext1": ["dep"], "ext2": ["dep"]} + ) + create_basic_wheel_for_package( + script, "pkg", "2.0", extras={"ext1": ["dep"], "ext2": ["dep"]} + ) + + to_install: tuple[str, str] = ( + "pkg[ext1]>1", "pkg[ext2]==1.0" if two_extras else "pkg==1.0" + ) + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + *(to_install if not swap_order else reversed(to_install)), + expect_error=True, + ) + assert "pkg-2.0" not in result.stdout or "pkg-1.0" not in result.stdout, ( + "Should only try one of 1.0, 2.0 depending on order" + ) + assert "looking at multiple versions" not in result.stdout, ( + "Should not have to look at multiple versions to conclude conflict" + ) + assert "conflict is caused by" in result.stdout, ( + "Resolver should be trivially able to find conflict cause" + ) + + def test_new_resolver_respect_user_requested_if_extra_is_installed( script: PipTestEnvironment, ) -> None: From 3fa373c0789699233726c02c2e72643d66da26e0 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 25 Jul 2023 15:59:20 +0200 Subject: [PATCH 031/156] added test for comes-from reporting --- tests/functional/test_new_resolver.py | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index 88dd635ae..e597669b3 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -2429,3 +2429,31 @@ def test_new_resolver_works_when_failing_package_builds_are_disallowed( ) script.assert_installed(pkg2="1.0", pkg1="1.0") + + +@pytest.mark.parametrize("swap_order", (True, False)) +def test_new_resolver_comes_from_with_extra( + script: PipTestEnvironment, swap_order: bool +) -> None: + """ + Verify that reporting where a dependency comes from is accurate when it comes + from a package with an extra. + + :param swap_order: swap the order the install specifiers appear in + """ + create_basic_wheel_for_package(script, "dep", "1.0") + create_basic_wheel_for_package(script, "pkg", "1.0", extras={"ext": ["dep"]}) + + to_install: tuple[str, str] = ("pkg", "pkg[ext]") + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + *(to_install if not swap_order else reversed(to_install)), + ) + assert "(from pkg[ext])" in result.stdout + assert "(from pkg)" not in result.stdout + script.assert_installed(pkg="1.0", dep="1.0") From e5690173515ce0dc82bcbc254d9211ca4124031c Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 25 Jul 2023 16:34:23 +0200 Subject: [PATCH 032/156] added test case for report bugfix --- tests/functional/test_install_report.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index 003b29d38..1c3ffe80b 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -64,14 +64,26 @@ def test_install_report_dep( assert _install_dict(report)["simple"]["requested"] is False +@pytest.mark.parametrize( + "specifiers", + [ + # result should be the same regardless of the method and order in which + # extras are specified + ("Paste[openid]==1.7.5.1",), + ("Paste==1.7.5.1", "Paste[openid]==1.7.5.1"), + ("Paste[openid]==1.7.5.1", "Paste==1.7.5.1"), + ], +) @pytest.mark.network -def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> None: +def test_install_report_index( + script: PipTestEnvironment, tmp_path: Path, specifiers: tuple[str, ...] +) -> None: """Test report for sdist obtained from index.""" report_path = tmp_path / "report.json" script.pip( "install", "--dry-run", - "Paste[openid]==1.7.5.1", + *specifiers, "--report", str(report_path), ) From cc6a2bded22001a6a3996f741b674ab1bab835ff Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 25 Jul 2023 16:38:51 +0200 Subject: [PATCH 033/156] added second report test case --- tests/functional/test_install_report.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index 1c3ffe80b..f9a2e27c0 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -105,6 +105,26 @@ def test_install_report_index( assert "requires_dist" in paste_report["metadata"] +@pytest.mark.network +def test_install_report_index_multiple_extras( + script: PipTestEnvironment, tmp_path: Path +) -> None: + """Test report for sdist obtained from index, with multiple extras requested.""" + report_path = tmp_path / "report.json" + script.pip( + "install", + "--dry-run", + "Paste[openid]", + "Paste[subprocess]", + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + install_dict = _install_dict(report) + assert "paste" in install_dict + assert install_dict["paste"]["requested_extras"] == ["openid", "subprocess"] + + @pytest.mark.network def test_install_report_direct_archive( script: PipTestEnvironment, tmp_path: Path, shared_data: TestData From 4ae829cb3f40d6a64c86988e2f591c3344123bcd Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 25 Jul 2023 17:14:50 +0200 Subject: [PATCH 034/156] news entries --- news/11924.bugfix.rst | 1 + news/12095.bugfix.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 news/11924.bugfix.rst create mode 100644 news/12095.bugfix.rst diff --git a/news/11924.bugfix.rst b/news/11924.bugfix.rst new file mode 100644 index 000000000..30bc60e6b --- /dev/null +++ b/news/11924.bugfix.rst @@ -0,0 +1 @@ +Improve extras resolution for multiple constraints on same base package. diff --git a/news/12095.bugfix.rst b/news/12095.bugfix.rst new file mode 100644 index 000000000..1f5018326 --- /dev/null +++ b/news/12095.bugfix.rst @@ -0,0 +1 @@ +Consistently report whether a dependency comes from an extra. From dc01a40d413351085410a39dc6f616c5e1e21002 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 25 Jul 2023 17:19:21 +0200 Subject: [PATCH 035/156] py38 compatibility --- src/pip/_internal/resolution/resolvelib/factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 03820edde..fdb5c4987 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -447,7 +447,7 @@ class Factory: def _make_requirements_from_install_req( self, ireq: InstallRequirement, requested_extras: Iterable[str] - ) -> list[Requirement]: + ) -> List[Requirement]: """ Returns requirement objects associated with the given InstallRequirement. In most cases this will be a single object but the following special cases exist: @@ -543,7 +543,7 @@ class Factory: specifier: str, comes_from: Optional[InstallRequirement], requested_extras: Iterable[str] = (), - ) -> list[Requirement]: + ) -> List[Requirement]: """ Returns requirement objects associated with the given specifier. In most cases this will be a single object but the following special cases exist: From 292387f20b8c6e0d57e9eec940621ef0932499c8 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 25 Jul 2023 17:25:05 +0200 Subject: [PATCH 036/156] py37 compatibility --- tests/functional/test_install_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index f9a2e27c0..d7553ec03 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -1,7 +1,7 @@ import json import textwrap from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, Tuple import pytest from packaging.utils import canonicalize_name @@ -76,7 +76,7 @@ def test_install_report_dep( ) @pytest.mark.network def test_install_report_index( - script: PipTestEnvironment, tmp_path: Path, specifiers: tuple[str, ...] + script: PipTestEnvironment, tmp_path: Path, specifiers: Tuple[str, ...] ) -> None: """Test report for sdist obtained from index.""" report_path = tmp_path / "report.json" From 39e1102800af8be86ed385aed7f93f6535262d29 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Tue, 25 Jul 2023 17:33:06 +0200 Subject: [PATCH 037/156] fixed minor type errors --- src/pip/_internal/req/constructors.py | 16 +++++++++++++--- .../_internal/resolution/resolvelib/factory.py | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index f97bded98..3b7243f82 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -64,7 +64,9 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme given requirement already has extras those are replaced (or dropped if no new extras are given). """ - match: re.Match = re.fullmatch(r"([^;\[<>~=]+)(\[[^\]]*\])?(.*)", str(req)) + match: Optional[re.Match[str]] = re.fullmatch( + r"([^;\[<>~=]+)(\[[^\]]*\])?(.*)", str(req) + ) # ireq.req is a valid requirement so the regex should always match assert match is not None pre: Optional[str] = match.group(1) @@ -534,7 +536,11 @@ def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement: (comes_from). """ return InstallRequirement( - req=_set_requirement_extras(ireq.req, set()), + req=( + _set_requirement_extras(ireq.req, set()) + if ireq.req is not None + else None + ), comes_from=ireq, editable=ireq.editable, link=ireq.link, @@ -561,5 +567,9 @@ def install_req_extend_extras( """ result = copy.copy(ireq) result.extras = {*ireq.extras, *extras} - result.req = _set_requirement_extras(ireq.req, result.extras) + result.req = ( + _set_requirement_extras(ireq.req, result.extras) + if ireq.req is not None + else None + ) return result diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index fdb5c4987..2812fab57 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -465,7 +465,7 @@ class Factory: ) return [] if not ireq.link: - if ireq.extras and ireq.req.specifier: + if ireq.extras and ireq.req is not None and ireq.req.specifier: return [ SpecifierRequirement(ireq, drop_extras=True), SpecifierRequirement(ireq), From e6333bb4d18edc8aec9b38601f81867ad1036807 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 10:32:58 +0200 Subject: [PATCH 038/156] linting --- src/pip/_internal/req/constructors.py | 12 +++------- .../resolution/resolvelib/requirements.py | 2 +- tests/functional/test_new_resolver.py | 24 ++++++++++--------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 3b7243f82..b5f176e6b 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -16,7 +16,7 @@ from typing import Collection, Dict, List, Optional, Set, Tuple, Union from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import InvalidRequirement, Requirement -from pip._vendor.packaging.specifiers import Specifier, SpecifierSet +from pip._vendor.packaging.specifiers import Specifier from pip._internal.exceptions import InstallationError from pip._internal.models.index import PyPI, TestPyPI @@ -72,11 +72,7 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme pre: Optional[str] = match.group(1) post: Optional[str] = match.group(3) assert pre is not None and post is not None - extras: str = ( - "[%s]" % ",".join(sorted(new_extras)) - if new_extras - else "" - ) + extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else "" return Requirement(pre + extras + post) @@ -537,9 +533,7 @@ def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement: """ return InstallRequirement( req=( - _set_requirement_extras(ireq.req, set()) - if ireq.req is not None - else None + _set_requirement_extras(ireq.req, set()) if ireq.req is not None else None ), comes_from=ireq, editable=ireq.editable, diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index e23b948ff..ad9892a17 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -1,8 +1,8 @@ from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import NormalizedName, canonicalize_name -from pip._internal.req.req_install import InstallRequirement from pip._internal.req.constructors import install_req_drop_extras +from pip._internal.req.req_install import InstallRequirement from .base import Candidate, CandidateLookup, Requirement, format_name diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index e597669b3..77dede2fc 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -2294,7 +2294,8 @@ def test_new_resolver_dont_backtrack_on_extra_if_base_constrained_in_requirement ) to_install: tuple[str, str] = ( - "pkg[ext1]", "pkg[ext2]==1.0" if two_extras else "pkg==1.0" + "pkg[ext1]", + "pkg[ext2]==1.0" if two_extras else "pkg==1.0", ) result = script.pip( @@ -2331,7 +2332,8 @@ def test_new_resolver_dont_backtrack_on_conflicting_constraints_on_extras( ) to_install: tuple[str, str] = ( - "pkg[ext1]>1", "pkg[ext2]==1.0" if two_extras else "pkg==1.0" + "pkg[ext1]>1", + "pkg[ext2]==1.0" if two_extras else "pkg==1.0", ) result = script.pip( @@ -2343,15 +2345,15 @@ def test_new_resolver_dont_backtrack_on_conflicting_constraints_on_extras( *(to_install if not swap_order else reversed(to_install)), expect_error=True, ) - assert "pkg-2.0" not in result.stdout or "pkg-1.0" not in result.stdout, ( - "Should only try one of 1.0, 2.0 depending on order" - ) - assert "looking at multiple versions" not in result.stdout, ( - "Should not have to look at multiple versions to conclude conflict" - ) - assert "conflict is caused by" in result.stdout, ( - "Resolver should be trivially able to find conflict cause" - ) + assert ( + "pkg-2.0" not in result.stdout or "pkg-1.0" not in result.stdout + ), "Should only try one of 1.0, 2.0 depending on order" + assert ( + "looking at multiple versions" not in result.stdout + ), "Should not have to look at multiple versions to conclude conflict" + assert ( + "conflict is caused by" in result.stdout + ), "Resolver should be trivially able to find conflict cause" def test_new_resolver_respect_user_requested_if_extra_is_installed( From 12073891776472dd3517106da1c26483d64e1557 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 10:33:43 +0200 Subject: [PATCH 039/156] made primary news fragment of type feature --- news/{11924.bugfix.rst => 11924.feature.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{11924.bugfix.rst => 11924.feature.rst} (100%) diff --git a/news/11924.bugfix.rst b/news/11924.feature.rst similarity index 100% rename from news/11924.bugfix.rst rename to news/11924.feature.rst From 6663b89a4d465f675b88bce52d4ad7cef9164c6a Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 10:37:00 +0200 Subject: [PATCH 040/156] added final bugfix news entry --- news/11924.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11924.bugfix.rst diff --git a/news/11924.bugfix.rst b/news/11924.bugfix.rst new file mode 100644 index 000000000..7a9ee3151 --- /dev/null +++ b/news/11924.bugfix.rst @@ -0,0 +1 @@ +Include all requested extras in the install report (``--report``). From 314d7c12549a60c8460b1e2a8dac82fe0cca848a Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 10:52:42 +0200 Subject: [PATCH 041/156] simplified regex --- src/pip/_internal/req/constructors.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index b5f176e6b..c03ae718e 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -65,7 +65,10 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme are given). """ match: Optional[re.Match[str]] = re.fullmatch( - r"([^;\[<>~=]+)(\[[^\]]*\])?(.*)", str(req) + # see https://peps.python.org/pep-0508/#complete-grammar + r"([\w\t .-]+)(\[[^\]]*\])?(.*)", + str(req), + flags=re.ASCII, ) # ireq.req is a valid requirement so the regex should always match assert match is not None From cc909e87e5ccada46b4eb8a2a90c329614dc9b01 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 11:06:24 +0200 Subject: [PATCH 042/156] reverted unnecessary changes --- src/pip/_internal/resolution/resolvelib/requirements.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index ad9892a17..becbd6c4b 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -56,7 +56,7 @@ class SpecifierRequirement(Requirement): self._extras = frozenset(self._ireq.extras) def __str__(self) -> str: - return str(self._ireq) + return str(self._ireq.req) def __repr__(self) -> str: return "{class_name}({requirement!r})".format( @@ -71,10 +71,7 @@ class SpecifierRequirement(Requirement): @property def name(self) -> str: - return format_name( - self.project_name, - self._extras, - ) + return format_name(self.project_name, self._extras) def format_for_error(self) -> str: # Convert comma-separated specifiers into "A, B, ..., F and G" From 6ed231a52be89295e3cecdb4f41eaf63f3152941 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 11:34:37 +0200 Subject: [PATCH 043/156] added unit tests for install req manipulation --- tests/unit/test_req.py | 87 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 545828f8e..b4819d832 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -6,7 +6,7 @@ import sys import tempfile from functools import partial from pathlib import Path -from typing import Iterator, Optional, Tuple, cast +from typing import Iterator, Optional, Set, Tuple, cast from unittest import mock import pytest @@ -33,6 +33,8 @@ from pip._internal.req import InstallRequirement, RequirementSet from pip._internal.req.constructors import ( _get_url_from_path, _looks_like_path, + install_req_drop_extras, + install_req_extend_extras, install_req_from_editable, install_req_from_line, install_req_from_parsed_requirement, @@ -763,6 +765,89 @@ class TestInstallRequirement: assert "appears to be a requirements file." in err_msg assert "If that is the case, use the '-r' flag to install" in err_msg + @pytest.mark.parametrize( + "inp, out", + [ + ("pkg", "pkg"), + ("pkg==1.0", "pkg==1.0"), + ("pkg ; python_version<='3.6'", "pkg"), + ("pkg[ext]", "pkg"), + ("pkg [ ext1, ext2 ]", "pkg"), + ("pkg [ ext1, ext2 ] @ https://example.com/", "pkg@ https://example.com/"), + ("pkg [ext] == 1.0; python_version<='3.6'", "pkg==1.0"), + ("pkg-all.allowed_chars0 ~= 2.0", "pkg-all.allowed_chars0~=2.0"), + ("pkg-all.allowed_chars0 [ext] ~= 2.0", "pkg-all.allowed_chars0~=2.0"), + ], + ) + def test_install_req_drop_extras(self, inp: str, out: str) -> None: + """ + Test behavior of install_req_drop_extras + """ + req = install_req_from_line(inp) + without_extras = install_req_drop_extras(req) + assert not without_extras.extras + assert str(without_extras.req) == out + # should always be a copy + assert req is not without_extras + assert req.req is not without_extras.req + # comes_from should point to original + assert without_extras.comes_from is req + # all else should be the same + assert without_extras.link == req.link + assert without_extras.markers == req.markers + assert without_extras.use_pep517 == req.use_pep517 + assert without_extras.isolated == req.isolated + assert without_extras.global_options == req.global_options + assert without_extras.hash_options == req.hash_options + assert without_extras.constraint == req.constraint + assert without_extras.config_settings == req.config_settings + assert without_extras.user_supplied == req.user_supplied + assert without_extras.permit_editable_wheels == req.permit_editable_wheels + + @pytest.mark.parametrize( + "inp, extras, out", + [ + ("pkg", {}, "pkg"), + ("pkg==1.0", {}, "pkg==1.0"), + ("pkg[ext]", {}, "pkg[ext]"), + ("pkg", {"ext"}, "pkg[ext]"), + ("pkg==1.0", {"ext"}, "pkg[ext]==1.0"), + ("pkg==1.0", {"ext1", "ext2"}, "pkg[ext1,ext2]==1.0"), + ("pkg; python_version<='3.6'", {"ext"}, "pkg[ext]"), + ("pkg[ext1,ext2]==1.0", {"ext2", "ext3"}, "pkg[ext1,ext2,ext3]==1.0"), + ( + "pkg-all.allowed_chars0 [ ext1 ] @ https://example.com/", + {"ext2"}, + "pkg-all.allowed_chars0[ext1,ext2]@ https://example.com/", + ), + ], + ) + def test_install_req_extend_extras( + self, inp: str, extras: Set[str], out: str + ) -> None: + """ + Test behavior of install_req_extend_extras + """ + req = install_req_from_line(inp) + extended = install_req_extend_extras(req, extras) + assert str(extended.req) == out + assert extended.req is not None + assert set(extended.extras) == set(extended.req.extras) + # should always be a copy + assert req is not extended + assert req.req is not extended.req + # all else should be the same + assert extended.link == req.link + assert extended.markers == req.markers + assert extended.use_pep517 == req.use_pep517 + assert extended.isolated == req.isolated + assert extended.global_options == req.global_options + assert extended.hash_options == req.hash_options + assert extended.constraint == req.constraint + assert extended.config_settings == req.config_settings + assert extended.user_supplied == req.user_supplied + assert extended.permit_editable_wheels == req.permit_editable_wheels + @mock.patch("pip._internal.req.req_install.os.path.abspath") @mock.patch("pip._internal.req.req_install.os.path.exists") From 55e9762873d824608ae52570ef3dcbb65e75f833 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 14:02:50 +0200 Subject: [PATCH 044/156] windows compatibility --- tests/lib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 7c06feaf3..d424a5e8d 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -645,7 +645,7 @@ class PipTestEnvironment(TestFileEnvironment): cwd = cwd or self.cwd if sys.platform == "win32": # Partial fix for ScriptTest.run using `shell=True` on Windows. - args = tuple(str(a).replace("^", "^^").replace("&", "^&") for a in args) + args = tuple(str(a).replace("^", "^^").replace("&", "^&").replace(">", "^>") for a in args) if allow_error: kw["expect_error"] = True From 504485c27644f7a8b44cec179bfc0fac181b0d9e Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 14:03:26 +0200 Subject: [PATCH 045/156] lint --- tests/lib/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index d424a5e8d..b6996f31d 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -645,7 +645,10 @@ class PipTestEnvironment(TestFileEnvironment): cwd = cwd or self.cwd if sys.platform == "win32": # Partial fix for ScriptTest.run using `shell=True` on Windows. - args = tuple(str(a).replace("^", "^^").replace("&", "^&").replace(">", "^>") for a in args) + args = tuple( + str(a).replace("^", "^^").replace("&", "^&").replace(">", "^>") + for a in args + ) if allow_error: kw["expect_error"] = True From f4a7c0c569caf665c11379cd9629e5a5163867f5 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 14:12:38 +0200 Subject: [PATCH 046/156] cleaned up windows fix --- tests/lib/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index b6996f31d..a7f2ade1a 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -645,10 +645,7 @@ class PipTestEnvironment(TestFileEnvironment): cwd = cwd or self.cwd if sys.platform == "win32": # Partial fix for ScriptTest.run using `shell=True` on Windows. - args = tuple( - str(a).replace("^", "^^").replace("&", "^&").replace(">", "^>") - for a in args - ) + args = tuple(re.sub("([&|()<>^])", r"^\1", str(a)) for a in args) if allow_error: kw["expect_error"] = True From 32e95be2130333e4f543778302fdc4d0c47043ad Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 26 Jul 2023 14:31:12 +0200 Subject: [PATCH 047/156] exclude brackets --- tests/lib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index a7f2ade1a..018152930 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -645,7 +645,7 @@ class PipTestEnvironment(TestFileEnvironment): cwd = cwd or self.cwd if sys.platform == "win32": # Partial fix for ScriptTest.run using `shell=True` on Windows. - args = tuple(re.sub("([&|()<>^])", r"^\1", str(a)) for a in args) + args = tuple(re.sub("([&|<>^])", r"^\1", str(a)) for a in args) if allow_error: kw["expect_error"] = True From b47f77d330bb7a8642af5490c0f03642f039974c Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon, 31 Jul 2023 20:21:23 -0400 Subject: [PATCH 048/156] add lots of comments on the function of BuildTracker --- src/pip/_internal/distributions/base.py | 12 +++++ src/pip/_internal/distributions/installed.py | 6 +++ src/pip/_internal/distributions/sdist.py | 8 ++- src/pip/_internal/distributions/wheel.py | 6 +++ .../operations/build/build_tracker.py | 49 ++++++++++++------- src/pip/_internal/operations/prepare.py | 10 ++-- 6 files changed, 68 insertions(+), 23 deletions(-) diff --git a/src/pip/_internal/distributions/base.py b/src/pip/_internal/distributions/base.py index 75ce2dc90..6fb0d7b77 100644 --- a/src/pip/_internal/distributions/base.py +++ b/src/pip/_internal/distributions/base.py @@ -1,4 +1,5 @@ import abc +from typing import Optional from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata.base import BaseDistribution @@ -19,12 +20,23 @@ class AbstractDistribution(metaclass=abc.ABCMeta): - we must be able to create a Distribution object exposing the above metadata. + + - if we need to do work in the build tracker, we must be able to generate a unique + string to identify the requirement in the build tracker. """ def __init__(self, req: InstallRequirement) -> None: super().__init__() self.req = req + @abc.abstractproperty + def build_tracker_id(self) -> Optional[str]: + """A string that uniquely identifies this requirement to the build tracker. + + If None, then this dist has no work to do in the build tracker, and + ``.prepare_distribution_metadata()`` will not be called.""" + raise NotImplementedError() + @abc.abstractmethod def get_metadata_distribution(self) -> BaseDistribution: raise NotImplementedError() diff --git a/src/pip/_internal/distributions/installed.py b/src/pip/_internal/distributions/installed.py index edb38aa1a..ab8d53be7 100644 --- a/src/pip/_internal/distributions/installed.py +++ b/src/pip/_internal/distributions/installed.py @@ -1,3 +1,5 @@ +from typing import Optional + from pip._internal.distributions.base import AbstractDistribution from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution @@ -10,6 +12,10 @@ class InstalledDistribution(AbstractDistribution): been computed. """ + @property + def build_tracker_id(self) -> Optional[str]: + return None + def get_metadata_distribution(self) -> BaseDistribution: assert self.req.satisfied_by is not None, "not actually installed" return self.req.satisfied_by diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index 4c2564793..15ff42b7b 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -1,5 +1,5 @@ import logging -from typing import Iterable, Set, Tuple +from typing import Iterable, Optional, Set, Tuple from pip._internal.build_env import BuildEnvironment from pip._internal.distributions.base import AbstractDistribution @@ -18,6 +18,12 @@ class SourceDistribution(AbstractDistribution): generated, either using PEP 517 or using the legacy `setup.py egg_info`. """ + @property + def build_tracker_id(self) -> Optional[str]: + """Identify this requirement uniquely by its link.""" + assert self.req.link + return self.req.link.url_without_fragment + def get_metadata_distribution(self) -> BaseDistribution: return self.req.get_dist() diff --git a/src/pip/_internal/distributions/wheel.py b/src/pip/_internal/distributions/wheel.py index 03aac775b..eb16e25cb 100644 --- a/src/pip/_internal/distributions/wheel.py +++ b/src/pip/_internal/distributions/wheel.py @@ -1,3 +1,5 @@ +from typing import Optional + from pip._vendor.packaging.utils import canonicalize_name from pip._internal.distributions.base import AbstractDistribution @@ -15,6 +17,10 @@ class WheelDistribution(AbstractDistribution): This does not need any preparation as wheels can be directly unpacked. """ + @property + def build_tracker_id(self) -> Optional[str]: + return None + def get_metadata_distribution(self) -> BaseDistribution: """Loads the metadata from the wheel file into memory and returns a Distribution that uses it, not relying on the wheel file or diff --git a/src/pip/_internal/operations/build/build_tracker.py b/src/pip/_internal/operations/build/build_tracker.py index 6621549b8..ffcdbbc03 100644 --- a/src/pip/_internal/operations/build/build_tracker.py +++ b/src/pip/_internal/operations/build/build_tracker.py @@ -51,10 +51,20 @@ def get_build_tracker() -> Generator["BuildTracker", None, None]: yield tracker +class TrackerId(str): + """Uniquely identifying string provided to the build tracker.""" + + class BuildTracker: + """Ensure that an sdist cannot request itself as a setup requirement. + + When an sdist is prepared, it identifies its setup requirements in the + context of ``BuildTracker#track()``. If a requirement shows up recursively, this + raises an exception. This stops fork bombs embedded in malicious packages.""" + def __init__(self, root: str) -> None: self._root = root - self._entries: Set[InstallRequirement] = set() + self._entries: Dict[TrackerId, InstallRequirement] = {} logger.debug("Created build tracker: %s", self._root) def __enter__(self) -> "BuildTracker": @@ -69,16 +79,15 @@ class BuildTracker: ) -> None: self.cleanup() - def _entry_path(self, link: Link) -> str: - hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() + def _entry_path(self, key: TrackerId) -> str: + hashed = hashlib.sha224(key.encode()).hexdigest() return os.path.join(self._root, hashed) - def add(self, req: InstallRequirement) -> None: + def add(self, req: InstallRequirement, key: TrackerId) -> None: """Add an InstallRequirement to build tracking.""" - assert req.link # Get the file to write information about this requirement. - entry_path = self._entry_path(req.link) + entry_path = self._entry_path(key) # Try reading from the file. If it exists and can be read from, a build # is already in progress, so a LookupError is raised. @@ -92,33 +101,37 @@ class BuildTracker: raise LookupError(message) # If we're here, req should really not be building already. - assert req not in self._entries + assert key not in self._entries # Start tracking this requirement. with open(entry_path, "w", encoding="utf-8") as fp: fp.write(str(req)) - self._entries.add(req) + self._entries[key] = req logger.debug("Added %s to build tracker %r", req, self._root) - def remove(self, req: InstallRequirement) -> None: + def remove(self, req: InstallRequirement, key: TrackerId) -> None: """Remove an InstallRequirement from build tracking.""" - assert req.link - # Delete the created file and the corresponding entries. - os.unlink(self._entry_path(req.link)) - self._entries.remove(req) + # Delete the created file and the corresponding entry. + os.unlink(self._entry_path(key)) + del self._entries[key] logger.debug("Removed %s from build tracker %r", req, self._root) def cleanup(self) -> None: - for req in set(self._entries): - self.remove(req) + for key, req in list(self._entries.items()): + self.remove(req, key) logger.debug("Removed build tracker: %r", self._root) @contextlib.contextmanager - def track(self, req: InstallRequirement) -> Generator[None, None, None]: - self.add(req) + def track(self, req: InstallRequirement, key: str) -> Generator[None, None, None]: + """Ensure that `key` cannot install itself as a setup requirement. + + :raises LookupError: If `key` was already provided in a parent invocation of + the context introduced by this method.""" + tracker_id = TrackerId(key) + self.add(req, tracker_id) yield - self.remove(req) + self.remove(req, tracker_id) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index c07b261fd..8402be01b 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -65,10 +65,12 @@ def _get_prepared_distribution( ) -> BaseDistribution: """Prepare a distribution for installation.""" abstract_dist = make_distribution_for_install_requirement(req) - with build_tracker.track(req): - abstract_dist.prepare_distribution_metadata( - finder, build_isolation, check_build_deps - ) + tracker_id = abstract_dist.build_tracker_id + if tracker_id is not None: + with build_tracker.track(req, tracker_id): + abstract_dist.prepare_distribution_metadata( + finder, build_isolation, check_build_deps + ) return abstract_dist.get_metadata_distribution() From 023b3d923746ffd4a7cafce471250af7daaa4fed Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue, 1 Aug 2023 13:59:42 -0400 Subject: [PATCH 049/156] add news --- news/12194.trivial.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12194.trivial.rst diff --git a/news/12194.trivial.rst b/news/12194.trivial.rst new file mode 100644 index 000000000..dfe5bbf1f --- /dev/null +++ b/news/12194.trivial.rst @@ -0,0 +1 @@ +Add lots of comments to the ``BuildTracker``. From d65ba2f6b6f446058b289b35373b0a6c36720fba Mon Sep 17 00:00:00 2001 From: Jeff Widman Date: Wed, 2 Aug 2023 13:54:12 -0700 Subject: [PATCH 050/156] Replace python2 deprecation with a badge of supported python versions The python world has (mostly) moved on from Python 2. Anyone not already aware of the py2->py3 migration is probably new to the ecosystem and started on Python 3. Additionally, it's convenient to see at a glance what versions of Python are supported by the current release. This pulls from PyPI versions, so will not immediately match `main` (we can probably change this to match `main` if preferred). So by doing this it's both more useful going forward, and also lets us drop the explicit notice about dropping Python 2 support. --- README.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index c2d5c9693..6ff117db5 100644 --- a/README.rst +++ b/README.rst @@ -3,9 +3,15 @@ pip - The Python Package Installer .. image:: https://img.shields.io/pypi/v/pip.svg :target: https://pypi.org/project/pip/ + :alt: PyPI + +.. image:: https://img.shields.io/pypi/pyversions/pip + :target: https://pypi.org/project/pip + :alt: PyPI - Python Version .. image:: https://readthedocs.org/projects/pip/badge/?version=latest :target: https://pip.pypa.io/en/latest + :alt: Documentation pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes. @@ -19,8 +25,6 @@ We release updates regularly, with a new version every 3 months. Find more detai * `Release notes`_ * `Release process`_ -**Note**: pip 21.0, in January 2021, removed Python 2 support, per pip's `Python 2 support policy`_. Please migrate to Python 3. - If you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms: * `Issue tracking`_ @@ -47,7 +51,6 @@ rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. .. _Release process: https://pip.pypa.io/en/latest/development/release-process/ .. _GitHub page: https://github.com/pypa/pip .. _Development documentation: https://pip.pypa.io/en/latest/development -.. _Python 2 support policy: https://pip.pypa.io/en/latest/development/release-process/#python-2-support .. _Issue tracking: https://github.com/pypa/pip/issues .. _Discourse channel: https://discuss.python.org/c/packaging .. _User IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa From d8cd93f4fa61df6be7786e301e4c17ab654d0107 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 8 Aug 2023 05:37:38 -0700 Subject: [PATCH 051/156] Fix incorrect use of re function in tests (#12213) --- news/732404DE-8011-4146-8CAD-85D7756D88A6.trivial.rst | 0 tests/lib/__init__.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 news/732404DE-8011-4146-8CAD-85D7756D88A6.trivial.rst diff --git a/news/732404DE-8011-4146-8CAD-85D7756D88A6.trivial.rst b/news/732404DE-8011-4146-8CAD-85D7756D88A6.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 7c06feaf3..b827f88ba 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -1187,7 +1187,7 @@ def create_basic_wheel_for_package( # Fix wheel distribution name by replacing runs of non-alphanumeric # characters with an underscore _ as per PEP 491 - name = re.sub(r"[^\w\d.]+", "_", name, re.UNICODE) + name = re.sub(r"[^\w\d.]+", "_", name) archive_name = f"{name}-{version}-py2.py3-none-any.whl" archive_path = script.scratch_path / archive_name From b2a151500b1e827de3ca950881204e4f4c9df08a Mon Sep 17 00:00:00 2001 From: Joshua Date: Tue, 8 Aug 2023 13:37:57 -0500 Subject: [PATCH 052/156] --dry-run is cool --- src/pip/_internal/cli/cmdoptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 02ba60827..64bc59bbd 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -92,10 +92,10 @@ def check_dist_restriction(options: Values, check_target: bool = False) -> None: ) if check_target: - if dist_restriction_set and not options.target_dir: + if not options.dry_run and dist_restriction_set and not options.target_dir: raise CommandError( "Can not use any platform or abi specific options unless " - "installing via '--target'" + "installing via '--target' or using '--dry-run'" ) From 38126ce5f811f98c2f45fcbfad47f3d5538120fe Mon Sep 17 00:00:00 2001 From: Joshua Date: Tue, 8 Aug 2023 13:42:11 -0500 Subject: [PATCH 053/156] HEAR YE HEAR YE --- news/12215.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12215.bugfix.rst diff --git a/news/12215.bugfix.rst b/news/12215.bugfix.rst new file mode 100644 index 000000000..f814540ff --- /dev/null +++ b/news/12215.bugfix.rst @@ -0,0 +1 @@ +Fix unnecessary error when using ``--dry-run`` and ``--python-version`` without ``--target`` From 864139adb0819053e76a3db4c50e916ef79cfc8c Mon Sep 17 00:00:00 2001 From: Joshua Date: Tue, 8 Aug 2023 14:32:59 -0500 Subject: [PATCH 054/156] add tests --- tests/functional/test_install.py | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index eabddfe58..038683246 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -2459,6 +2459,40 @@ def test_install_pip_prints_req_chain_local(script: PipTestEnvironment) -> None: ) +def test_install_dist_restriction_without_target(script: PipTestEnvironment) -> None: + result = script.pip( + "install", "--python-version=3.1", "--only-binary=:all:", expect_error=True + ) + assert ( + "Can not use any platform or abi specific options unless installing " + "via '--target'" in result.stderr + ), str(result) + + +def test_install_dist_restriction_dry_run_doesnt_require_target( + script: PipTestEnvironment, +) -> None: + create_basic_wheel_for_package( + script, + "base", + "0.1.0", + ) + + result = script.pip( + "install", + "--python-version=3.1", + "--only-binary=:all:", + "--dry-run", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "base", + ) + + assert not result.stderr, str(result) + + @pytest.mark.network def test_install_pip_prints_req_chain_pypi(script: PipTestEnvironment) -> None: """ From 46754f1580d46aaa11147796336cd79ea1682caa Mon Sep 17 00:00:00 2001 From: Joshua Date: Wed, 9 Aug 2023 09:04:26 -0500 Subject: [PATCH 055/156] It's not a bug(fix) it's a FEATURE --- news/12215.bugfix.rst | 1 - news/12215.feature.rst | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 news/12215.bugfix.rst create mode 100644 news/12215.feature.rst diff --git a/news/12215.bugfix.rst b/news/12215.bugfix.rst deleted file mode 100644 index f814540ff..000000000 --- a/news/12215.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix unnecessary error when using ``--dry-run`` and ``--python-version`` without ``--target`` diff --git a/news/12215.feature.rst b/news/12215.feature.rst new file mode 100644 index 000000000..407dc903e --- /dev/null +++ b/news/12215.feature.rst @@ -0,0 +1 @@ +Allow ``pip install --dry-run`` to use platform and ABI overriding options similar to ``--target``. From b1fd3ac3e483b59ce15b1006a0a757f4502864cb Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri, 11 Aug 2023 04:37:31 -0400 Subject: [PATCH 056/156] Update src/pip/_internal/operations/build/build_tracker.py Co-authored-by: Paul Moore --- src/pip/_internal/operations/build/build_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/build/build_tracker.py b/src/pip/_internal/operations/build/build_tracker.py index ffcdbbc03..d4cdcb89d 100644 --- a/src/pip/_internal/operations/build/build_tracker.py +++ b/src/pip/_internal/operations/build/build_tracker.py @@ -59,7 +59,7 @@ class BuildTracker: """Ensure that an sdist cannot request itself as a setup requirement. When an sdist is prepared, it identifies its setup requirements in the - context of ``BuildTracker#track()``. If a requirement shows up recursively, this + context of ``BuildTracker.track()``. If a requirement shows up recursively, this raises an exception. This stops fork bombs embedded in malicious packages.""" def __init__(self, root: str) -> None: From 4a853ea34831e9fa9695cd966905cd5b591e0b01 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri, 11 Aug 2023 04:38:07 -0400 Subject: [PATCH 057/156] Update src/pip/_internal/operations/build/build_tracker.py Co-authored-by: Paul Moore --- src/pip/_internal/operations/build/build_tracker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/build/build_tracker.py b/src/pip/_internal/operations/build/build_tracker.py index d4cdcb89d..37919322b 100644 --- a/src/pip/_internal/operations/build/build_tracker.py +++ b/src/pip/_internal/operations/build/build_tracker.py @@ -60,7 +60,9 @@ class BuildTracker: When an sdist is prepared, it identifies its setup requirements in the context of ``BuildTracker.track()``. If a requirement shows up recursively, this - raises an exception. This stops fork bombs embedded in malicious packages.""" + raises an exception. + + This stops fork bombs embedded in malicious packages.""" def __init__(self, root: str) -> None: self._root = root From 8e305f262fbba1e4a1555efdaee982877d235012 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon, 31 Jul 2023 04:05:45 -0400 Subject: [PATCH 058/156] add test for the *existing* `install --dry-run` functionality --- tests/functional/test_install.py | 67 +++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index eabddfe58..56efe2a5c 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -7,7 +7,7 @@ import sysconfig import textwrap from os.path import curdir, join, pardir from pathlib import Path -from typing import Dict, List, Tuple +from typing import Callable, Dict, Iterable, List, Optional, Tuple import pytest @@ -20,6 +20,7 @@ from tests.lib import ( PipTestEnvironment, ResolverVariant, TestData, + TestPipResult, _create_svn_repo, _create_test_package, create_basic_wheel_for_package, @@ -2371,14 +2372,68 @@ def test_install_logs_pip_version_in_debug( assert_re_match(pattern, result.stdout) -def test_install_dry_run(script: PipTestEnvironment, data: TestData) -> None: - """Test that pip install --dry-run logs what it would install.""" - result = script.pip( - "install", "--dry-run", "--find-links", data.find_links, "simple" - ) +@pytest.fixture +def install_find_links( + script: PipTestEnvironment, + data: TestData, +) -> Callable[[Iterable[str], bool, Optional[Path]], TestPipResult]: + def install( + args: Iterable[str], dry_run: bool, target_dir: Optional[Path] + ) -> TestPipResult: + return script.pip( + "install", + *( + ( + "--target", + str(target_dir), + ) + if target_dir is not None + else () + ), + *(("--dry-run",) if dry_run else ()), + "--no-index", + "--find-links", + data.find_links, + *args, + ) + + return install + + +@pytest.mark.parametrize( + "with_target_dir", + (True, False), +) +def test_install_dry_run_nothing_installed( + script: PipTestEnvironment, + tmpdir: Path, + install_find_links: Callable[[Iterable[str], bool, Optional[Path]], TestPipResult], + with_target_dir: bool, +) -> None: + """Test that pip install --dry-run logs what it would install, but doesn't actually + install anything.""" + if with_target_dir: + install_dir = tmpdir / "fake-install" + install_dir.mkdir() + else: + install_dir = None + + result = install_find_links(["simple"], True, install_dir) assert "Would install simple-3.0" in result.stdout assert "Successfully installed" not in result.stdout + script.assert_not_installed("simple") + if with_target_dir: + assert not os.listdir(install_dir) + + # Ensure that the same install command would normally have worked if not for + # --dry-run. + install_find_links(["simple"], False, install_dir) + if with_target_dir: + assert os.listdir(install_dir) + else: + script.assert_installed(simple="3.0") + @pytest.mark.skipif( sys.version_info < (3, 11), From e86da734c7a1bd909ff8c911a276f8f2e6df9348 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon, 31 Jul 2023 05:41:42 -0400 Subject: [PATCH 059/156] add test for hash mismatch --- tests/functional/test_fast_deps.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/functional/test_fast_deps.py b/tests/functional/test_fast_deps.py index 0109db825..b76b833b9 100644 --- a/tests/functional/test_fast_deps.py +++ b/tests/functional/test_fast_deps.py @@ -2,12 +2,14 @@ import fnmatch import json import os import pathlib +import re from os.path import basename from typing import Iterable from pip._vendor.packaging.utils import canonicalize_name from pytest import mark +from pip._internal.utils.misc import hash_file from tests.lib import PipTestEnvironment, TestData, TestPipResult @@ -101,3 +103,31 @@ def test_hash_mismatch(script: PipTestEnvironment, tmp_path: pathlib.Path) -> No expect_error=True, ) assert "DO NOT MATCH THE HASHES" in result.stderr + + +@mark.network +def test_hash_mismatch_existing_download( + script: PipTestEnvironment, tmp_path: pathlib.Path +) -> None: + reqs = tmp_path / "requirements.txt" + reqs.write_text("idna==2.10") + dl_dir = tmp_path / "downloads" + dl_dir.mkdir() + idna_wheel = dl_dir / "idna-2.10-py2.py3-none-any.whl" + idna_wheel.write_text("asdf") + result = script.pip( + "download", + "--use-feature=fast-deps", + "-r", + str(reqs), + "-d", + str(dl_dir), + allow_stderr_warning=True, + ) + assert re.search( + r"WARNING: Previously-downloaded file.*has bad hash", result.stderr + ) + assert ( + hash_file(str(idna_wheel))[0].hexdigest() + == "b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ) From 67ff36b838f543037169828be95f86f903d609c6 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon, 31 Jul 2023 04:07:55 -0400 Subject: [PATCH 060/156] move directory metadata test out of req install tests --- tests/unit/metadata/test_metadata.py | 14 ++++++++++++++ tests/unit/test_req.py | 17 ----------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/unit/metadata/test_metadata.py b/tests/unit/metadata/test_metadata.py index f77178fb9..47093fb54 100644 --- a/tests/unit/metadata/test_metadata.py +++ b/tests/unit/metadata/test_metadata.py @@ -129,3 +129,17 @@ def test_dist_found_in_zip(tmp_path: Path) -> None: dist = get_environment([location]).get_distribution("pkg") assert dist is not None and dist.location is not None assert Path(dist.location) == Path(location) + + +@pytest.mark.parametrize( + "path", + ( + "/path/to/foo.egg-info".replace("/", os.path.sep), + # Tests issue fixed by https://github.com/pypa/pip/pull/2530 + "/path/to/foo.egg-info/".replace("/", os.path.sep), + ), +) +def test_trailing_slash_directory_metadata(path: str) -> None: + dist = get_directory_distribution(path) + assert dist.raw_name == dist.canonical_name == "foo" + assert dist.location == "/path/to".replace("/", os.path.sep) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 545828f8e..2d1fa2694 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -23,7 +23,6 @@ from pip._internal.exceptions import ( PreviousBuildDirError, ) from pip._internal.index.package_finder import PackageFinder -from pip._internal.metadata import select_backend from pip._internal.models.direct_url import ArchiveInfo, DirectUrl, DirInfo, VcsInfo from pip._internal.models.link import Link from pip._internal.network.session import PipSession @@ -600,22 +599,6 @@ class TestInstallRequirement: assert req.link is not None assert req.link.url == url - @pytest.mark.parametrize( - "path", - ( - "/path/to/foo.egg-info".replace("/", os.path.sep), - # Tests issue fixed by https://github.com/pypa/pip/pull/2530 - "/path/to/foo.egg-info/".replace("/", os.path.sep), - ), - ) - def test_get_dist(self, path: str) -> None: - req = install_req_from_line("foo") - req.metadata_directory = path - dist = req.get_dist() - assert isinstance(dist, select_backend().Distribution) - assert dist.raw_name == dist.canonical_name == "foo" - assert dist.location == "/path/to".replace("/", os.path.sep) - def test_markers(self) -> None: for line in ( # recommended syntax From 20b54de4dfb3567e088ca058819de43df90364d9 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue, 1 Aug 2023 13:34:50 -0400 Subject: [PATCH 061/156] add news --- news/12183.trivial.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12183.trivial.rst diff --git a/news/12183.trivial.rst b/news/12183.trivial.rst new file mode 100644 index 000000000..c22e854c9 --- /dev/null +++ b/news/12183.trivial.rst @@ -0,0 +1 @@ +Add test cases for some behaviors of ``install --dry-run`` and ``--use-feature=fast-deps``. From e27af2c3c9a336d6379b24f92cd98cb14c1bd090 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:46:22 -0400 Subject: [PATCH 062/156] add notes on hash mismatch testing --- tests/functional/test_fast_deps.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/functional/test_fast_deps.py b/tests/functional/test_fast_deps.py index b76b833b9..9e529c089 100644 --- a/tests/functional/test_fast_deps.py +++ b/tests/functional/test_fast_deps.py @@ -106,9 +106,12 @@ def test_hash_mismatch(script: PipTestEnvironment, tmp_path: pathlib.Path) -> No @mark.network -def test_hash_mismatch_existing_download( +def test_hash_mismatch_existing_download_for_metadata_only_wheel( script: PipTestEnvironment, tmp_path: pathlib.Path ) -> None: + """Metadata-only wheels from PEP 658 or fast-deps check for hash matching in + a separate code path than when the wheel is downloaded all at once. Make sure we + still check for hash mismatches.""" reqs = tmp_path / "requirements.txt" reqs.write_text("idna==2.10") dl_dir = tmp_path / "downloads" @@ -117,6 +120,7 @@ def test_hash_mismatch_existing_download( idna_wheel.write_text("asdf") result = script.pip( "download", + # Ensure that we have a metadata-only dist for idna. "--use-feature=fast-deps", "-r", str(reqs), @@ -127,6 +131,7 @@ def test_hash_mismatch_existing_download( assert re.search( r"WARNING: Previously-downloaded file.*has bad hash", result.stderr ) + # This is the correct hash for idna==2.10. assert ( hash_file(str(idna_wheel))[0].hexdigest() == "b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" From 8704c7a5dbd62367b01f39317b6213578683f4f6 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:51:33 -0400 Subject: [PATCH 063/156] remove unnecessary fixture --- tests/functional/test_install.py | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 56efe2a5c..5e8a82fb3 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -7,7 +7,7 @@ import sysconfig import textwrap from os.path import curdir, join, pardir from pathlib import Path -from typing import Callable, Dict, Iterable, List, Optional, Tuple +from typing import Dict, Iterable, List, Optional, Tuple import pytest @@ -2372,32 +2372,30 @@ def test_install_logs_pip_version_in_debug( assert_re_match(pattern, result.stdout) -@pytest.fixture def install_find_links( script: PipTestEnvironment, data: TestData, -) -> Callable[[Iterable[str], bool, Optional[Path]], TestPipResult]: - def install( - args: Iterable[str], dry_run: bool, target_dir: Optional[Path] - ) -> TestPipResult: - return script.pip( - "install", - *( - ( - "--target", - str(target_dir), - ) - if target_dir is not None - else () - ), - *(("--dry-run",) if dry_run else ()), - "--no-index", - "--find-links", - data.find_links, - *args, - ) - - return install + args: Iterable[str], + *, + dry_run: bool, + target_dir: Optional[Path], +) -> TestPipResult: + return script.pip( + "install", + *( + ( + "--target", + str(target_dir), + ) + if target_dir is not None + else () + ), + *(("--dry-run",) if dry_run else ()), + "--no-index", + "--find-links", + data.find_links, + *args, + ) @pytest.mark.parametrize( @@ -2406,8 +2404,8 @@ def install_find_links( ) def test_install_dry_run_nothing_installed( script: PipTestEnvironment, + data: TestData, tmpdir: Path, - install_find_links: Callable[[Iterable[str], bool, Optional[Path]], TestPipResult], with_target_dir: bool, ) -> None: """Test that pip install --dry-run logs what it would install, but doesn't actually @@ -2418,7 +2416,9 @@ def test_install_dry_run_nothing_installed( else: install_dir = None - result = install_find_links(["simple"], True, install_dir) + result = install_find_links( + script, data, ["simple"], dry_run=True, target_dir=install_dir + ) assert "Would install simple-3.0" in result.stdout assert "Successfully installed" not in result.stdout @@ -2428,7 +2428,7 @@ def test_install_dry_run_nothing_installed( # Ensure that the same install command would normally have worked if not for # --dry-run. - install_find_links(["simple"], False, install_dir) + install_find_links(script, data, ["simple"], dry_run=False, target_dir=install_dir) if with_target_dir: assert os.listdir(install_dir) else: From 2e365bdab1a37c296eae264991c2e88d57f73a67 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue, 1 Aug 2023 22:11:07 -0400 Subject: [PATCH 064/156] move test_download_metadata mock pypi index utilities to conftest.py --- tests/conftest.py | 242 +++++++++++++++++++++++ tests/functional/test_download.py | 311 +++++------------------------- 2 files changed, 286 insertions(+), 267 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a183cadf2..f481e06c8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1,32 @@ import compileall import fnmatch +import http.server import io import os import re import shutil import subprocess import sys +import threading from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from enum import Enum +from hashlib import sha256 from pathlib import Path +from textwrap import dedent from typing import ( TYPE_CHECKING, + Any, AnyStr, Callable, + ClassVar, Dict, Iterable, Iterator, List, Optional, + Set, + Tuple, Union, ) from unittest.mock import patch @@ -750,3 +760,235 @@ def proxy(request: pytest.FixtureRequest) -> str: @pytest.fixture def enable_user_site(virtualenv: VirtualEnvironment) -> None: virtualenv.user_site_packages = True + + +class MetadataKind(Enum): + """All the types of values we might be provided for the data-dist-info-metadata + attribute from PEP 658.""" + + # Valid: will read metadata from the dist instead. + No = "none" + # Valid: will read the .metadata file, but won't check its hash. + Unhashed = "unhashed" + # Valid: will read the .metadata file and check its hash matches. + Sha256 = "sha256" + # Invalid: will error out after checking the hash. + WrongHash = "wrong-hash" + # Invalid: will error out after failing to fetch the .metadata file. + NoFile = "no-file" + + +@dataclass(frozen=True) +class FakePackage: + """Mock package structure used to generate a PyPI repository. + + FakePackage name and version should correspond to sdists (.tar.gz files) in our test + data.""" + + name: str + version: str + filename: str + metadata: MetadataKind + # This will override any dependencies specified in the actual dist's METADATA. + requires_dist: Tuple[str, ...] = () + # This will override the Name specified in the actual dist's METADATA. + metadata_name: Optional[str] = None + + def metadata_filename(self) -> str: + """This is specified by PEP 658.""" + return f"{self.filename}.metadata" + + def generate_additional_tag(self) -> str: + """This gets injected into the tag in the generated PyPI index page for this + package.""" + if self.metadata == MetadataKind.No: + return "" + if self.metadata in [MetadataKind.Unhashed, MetadataKind.NoFile]: + return 'data-dist-info-metadata="true"' + if self.metadata == MetadataKind.WrongHash: + return 'data-dist-info-metadata="sha256=WRONG-HASH"' + assert self.metadata == MetadataKind.Sha256 + checksum = sha256(self.generate_metadata()).hexdigest() + return f'data-dist-info-metadata="sha256={checksum}"' + + def requires_str(self) -> str: + if not self.requires_dist: + return "" + joined = " and ".join(self.requires_dist) + return f"Requires-Dist: {joined}" + + def generate_metadata(self) -> bytes: + """This is written to `self.metadata_filename()` and will override the actual + dist's METADATA, unless `self.metadata == MetadataKind.NoFile`.""" + return dedent( + f"""\ + Metadata-Version: 2.1 + Name: {self.metadata_name or self.name} + Version: {self.version} + {self.requires_str()} + """ + ).encode("utf-8") + + +@pytest.fixture(scope="session") +def fake_packages() -> Dict[str, List[FakePackage]]: + """The package database we generate for testing PEP 658 support.""" + return { + "simple": [ + FakePackage("simple", "1.0", "simple-1.0.tar.gz", MetadataKind.Sha256), + FakePackage("simple", "2.0", "simple-2.0.tar.gz", MetadataKind.No), + # This will raise a hashing error. + FakePackage("simple", "3.0", "simple-3.0.tar.gz", MetadataKind.WrongHash), + ], + "simple2": [ + # Override the dependencies here in order to force pip to download + # simple-1.0.tar.gz as well. + FakePackage( + "simple2", + "1.0", + "simple2-1.0.tar.gz", + MetadataKind.Unhashed, + ("simple==1.0",), + ), + # This will raise an error when pip attempts to fetch the metadata file. + FakePackage("simple2", "2.0", "simple2-2.0.tar.gz", MetadataKind.NoFile), + # This has a METADATA file with a mismatched name. + FakePackage( + "simple2", + "3.0", + "simple2-3.0.tar.gz", + MetadataKind.Sha256, + metadata_name="not-simple2", + ), + ], + "colander": [ + # Ensure we can read the dependencies from a metadata file within a wheel + # *without* PEP 658 metadata. + FakePackage( + "colander", + "0.9.9", + "colander-0.9.9-py2.py3-none-any.whl", + MetadataKind.No, + ), + ], + "compilewheel": [ + # Ensure we can override the dependencies of a wheel file by injecting PEP + # 658 metadata. + FakePackage( + "compilewheel", + "1.0", + "compilewheel-1.0-py2.py3-none-any.whl", + MetadataKind.Unhashed, + ("simple==1.0",), + ), + ], + "has-script": [ + # Ensure we check PEP 658 metadata hashing errors for wheel files. + FakePackage( + "has-script", + "1.0", + "has.script-1.0-py2.py3-none-any.whl", + MetadataKind.WrongHash, + ), + ], + "translationstring": [ + FakePackage( + "translationstring", + "1.1", + "translationstring-1.1.tar.gz", + MetadataKind.No, + ), + ], + "priority": [ + # Ensure we check for a missing metadata file for wheels. + FakePackage( + "priority", + "1.0", + "priority-1.0-py2.py3-none-any.whl", + MetadataKind.NoFile, + ), + ], + "requires-simple-extra": [ + # Metadata name is not canonicalized. + FakePackage( + "requires-simple-extra", + "0.1", + "requires_simple_extra-0.1-py2.py3-none-any.whl", + MetadataKind.Sha256, + metadata_name="Requires_Simple.Extra", + ), + ], + } + + +@pytest.fixture(scope="session") +def html_index_for_packages( + shared_data: TestData, + fake_packages: Dict[str, List[FakePackage]], + tmpdir_factory: pytest.TempPathFactory, +) -> Path: + """Generate a PyPI HTML package index within a local directory pointing to + synthetic test data.""" + html_dir = tmpdir_factory.mktemp("fake_index_html_content") + + # (1) Generate the content for a PyPI index.html. + pkg_links = "\n".join( + f' {pkg}' for pkg in fake_packages.keys() + ) + index_html = f"""\ + + + + + Simple index + + +{pkg_links} + +""" + # (2) Generate the index.html in a new subdirectory of the temp directory. + (html_dir / "index.html").write_text(index_html) + + # (3) Generate subdirectories for individual packages, each with their own + # index.html. + for pkg, links in fake_packages.items(): + pkg_subdir = html_dir / pkg + pkg_subdir.mkdir() + + download_links: List[str] = [] + for package_link in links: + # (3.1) Generate the tag which pip can crawl pointing to this + # specific package version. + download_links.append( + f' {package_link.filename}
' # noqa: E501 + ) + # (3.2) Copy over the corresponding file in `shared_data.packages`. + shutil.copy( + shared_data.packages / package_link.filename, + pkg_subdir / package_link.filename, + ) + # (3.3) Write a metadata file, if applicable. + if package_link.metadata != MetadataKind.NoFile: + with open(pkg_subdir / package_link.metadata_filename(), "wb") as f: + f.write(package_link.generate_metadata()) + + # (3.4) After collating all the download links and copying over the files, + # write an index.html with the generated download links for each + # copied file for this specific package name. + download_links_str = "\n".join(download_links) + pkg_index_content = f"""\ + + + + + Links for {pkg} + + +

Links for {pkg}

+{download_links_str} + +""" + with open(pkg_subdir / "index.html", "w") as f: + f.write(pkg_index_content) + + return html_dir diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 8da185c06..bedadc704 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -1,14 +1,11 @@ +import http.server import os import re import shutil import textwrap -import uuid -from dataclasses import dataclass -from enum import Enum from hashlib import sha256 from pathlib import Path -from textwrap import dedent -from typing import Callable, Dict, List, Optional, Tuple +from typing import Callable, List, Tuple import pytest @@ -1237,181 +1234,16 @@ def test_download_use_pep517_propagation( assert len(downloads) == 2 -class MetadataKind(Enum): - """All the types of values we might be provided for the data-dist-info-metadata - attribute from PEP 658.""" - - # Valid: will read metadata from the dist instead. - No = "none" - # Valid: will read the .metadata file, but won't check its hash. - Unhashed = "unhashed" - # Valid: will read the .metadata file and check its hash matches. - Sha256 = "sha256" - # Invalid: will error out after checking the hash. - WrongHash = "wrong-hash" - # Invalid: will error out after failing to fetch the .metadata file. - NoFile = "no-file" - - -@dataclass(frozen=True) -class Package: - """Mock package structure used to generate a PyPI repository. - - Package name and version should correspond to sdists (.tar.gz files) in our test - data.""" - - name: str - version: str - filename: str - metadata: MetadataKind - # This will override any dependencies specified in the actual dist's METADATA. - requires_dist: Tuple[str, ...] = () - # This will override the Name specified in the actual dist's METADATA. - metadata_name: Optional[str] = None - - def metadata_filename(self) -> str: - """This is specified by PEP 658.""" - return f"{self.filename}.metadata" - - def generate_additional_tag(self) -> str: - """This gets injected into the tag in the generated PyPI index page for this - package.""" - if self.metadata == MetadataKind.No: - return "" - if self.metadata in [MetadataKind.Unhashed, MetadataKind.NoFile]: - return 'data-dist-info-metadata="true"' - if self.metadata == MetadataKind.WrongHash: - return 'data-dist-info-metadata="sha256=WRONG-HASH"' - assert self.metadata == MetadataKind.Sha256 - checksum = sha256(self.generate_metadata()).hexdigest() - return f'data-dist-info-metadata="sha256={checksum}"' - - def requires_str(self) -> str: - if not self.requires_dist: - return "" - joined = " and ".join(self.requires_dist) - return f"Requires-Dist: {joined}" - - def generate_metadata(self) -> bytes: - """This is written to `self.metadata_filename()` and will override the actual - dist's METADATA, unless `self.metadata == MetadataKind.NoFile`.""" - return dedent( - f"""\ - Metadata-Version: 2.1 - Name: {self.metadata_name or self.name} - Version: {self.version} - {self.requires_str()} - """ - ).encode("utf-8") - - @pytest.fixture(scope="function") -def write_index_html_content(tmpdir: Path) -> Callable[[str], Path]: - """Generate a PyPI package index.html within a temporary local directory.""" - html_dir = tmpdir / "index_html_content" - html_dir.mkdir() - - def generate_index_html_subdir(index_html: str) -> Path: - """Create a new subdirectory after a UUID and write an index.html.""" - new_subdir = html_dir / uuid.uuid4().hex - new_subdir.mkdir() - - with open(new_subdir / "index.html", "w") as f: - f.write(index_html) - - return new_subdir - - return generate_index_html_subdir - - -@pytest.fixture(scope="function") -def html_index_for_packages( - shared_data: TestData, - write_index_html_content: Callable[[str], Path], -) -> Callable[..., Path]: - """Generate a PyPI HTML package index within a local directory pointing to - blank data.""" - - def generate_html_index_for_packages(packages: Dict[str, List[Package]]) -> Path: - """ - Produce a PyPI directory structure pointing to the specified packages. - """ - # (1) Generate the content for a PyPI index.html. - pkg_links = "\n".join( - f' {pkg}' for pkg in packages.keys() - ) - index_html = f"""\ - - - - - Simple index - - -{pkg_links} - -""" - # (2) Generate the index.html in a new subdirectory of the temp directory. - index_html_subdir = write_index_html_content(index_html) - - # (3) Generate subdirectories for individual packages, each with their own - # index.html. - for pkg, links in packages.items(): - pkg_subdir = index_html_subdir / pkg - pkg_subdir.mkdir() - - download_links: List[str] = [] - for package_link in links: - # (3.1) Generate the tag which pip can crawl pointing to this - # specific package version. - download_links.append( - f' {package_link.filename}
' # noqa: E501 - ) - # (3.2) Copy over the corresponding file in `shared_data.packages`. - shutil.copy( - shared_data.packages / package_link.filename, - pkg_subdir / package_link.filename, - ) - # (3.3) Write a metadata file, if applicable. - if package_link.metadata != MetadataKind.NoFile: - with open(pkg_subdir / package_link.metadata_filename(), "wb") as f: - f.write(package_link.generate_metadata()) - - # (3.4) After collating all the download links and copying over the files, - # write an index.html with the generated download links for each - # copied file for this specific package name. - download_links_str = "\n".join(download_links) - pkg_index_content = f"""\ - - - - - Links for {pkg} - - -

Links for {pkg}

-{download_links_str} - -""" - with open(pkg_subdir / "index.html", "w") as f: - f.write(pkg_index_content) - - return index_html_subdir - - return generate_html_index_for_packages - - -@pytest.fixture(scope="function") -def download_generated_html_index( +def download_local_html_index( script: PipTestEnvironment, - html_index_for_packages: Callable[[Dict[str, List[Package]]], Path], + html_index_for_packages: Path, tmpdir: Path, ) -> Callable[..., Tuple[TestPipResult, Path]]: """Execute `pip download` against a generated PyPI index.""" download_dir = tmpdir / "download_dir" def run_for_generated_index( - packages: Dict[str, List[Package]], args: List[str], allow_error: bool = False, ) -> Tuple[TestPipResult, Path]: @@ -1419,13 +1251,12 @@ def download_generated_html_index( Produce a PyPI directory structure pointing to the specified packages, then execute `pip download -i ...` pointing to our generated index. """ - index_dir = html_index_for_packages(packages) pip_args = [ "download", "-d", str(download_dir), "-i", - path_to_url(str(index_dir)), + path_to_url(str(html_index_for_packages)), *args, ] result = script.pip(*pip_args, allow_error=allow_error) @@ -1434,84 +1265,35 @@ def download_generated_html_index( return run_for_generated_index -# The package database we generate for testing PEP 658 support. -_simple_packages: Dict[str, List[Package]] = { - "simple": [ - Package("simple", "1.0", "simple-1.0.tar.gz", MetadataKind.Sha256), - Package("simple", "2.0", "simple-2.0.tar.gz", MetadataKind.No), - # This will raise a hashing error. - Package("simple", "3.0", "simple-3.0.tar.gz", MetadataKind.WrongHash), - ], - "simple2": [ - # Override the dependencies here in order to force pip to download - # simple-1.0.tar.gz as well. - Package( - "simple2", - "1.0", - "simple2-1.0.tar.gz", - MetadataKind.Unhashed, - ("simple==1.0",), - ), - # This will raise an error when pip attempts to fetch the metadata file. - Package("simple2", "2.0", "simple2-2.0.tar.gz", MetadataKind.NoFile), - # This has a METADATA file with a mismatched name. - Package( - "simple2", - "3.0", - "simple2-3.0.tar.gz", - MetadataKind.Sha256, - metadata_name="not-simple2", - ), - ], - "colander": [ - # Ensure we can read the dependencies from a metadata file within a wheel - # *without* PEP 658 metadata. - Package( - "colander", "0.9.9", "colander-0.9.9-py2.py3-none-any.whl", MetadataKind.No - ), - ], - "compilewheel": [ - # Ensure we can override the dependencies of a wheel file by injecting PEP - # 658 metadata. - Package( - "compilewheel", - "1.0", - "compilewheel-1.0-py2.py3-none-any.whl", - MetadataKind.Unhashed, - ("simple==1.0",), - ), - ], - "has-script": [ - # Ensure we check PEP 658 metadata hashing errors for wheel files. - Package( - "has-script", - "1.0", - "has.script-1.0-py2.py3-none-any.whl", - MetadataKind.WrongHash, - ), - ], - "translationstring": [ - Package( - "translationstring", "1.1", "translationstring-1.1.tar.gz", MetadataKind.No - ), - ], - "priority": [ - # Ensure we check for a missing metadata file for wheels. - Package( - "priority", "1.0", "priority-1.0-py2.py3-none-any.whl", MetadataKind.NoFile - ), - ], - "requires-simple-extra": [ - # Metadata name is not canonicalized. - Package( - "requires-simple-extra", - "0.1", - "requires_simple_extra-0.1-py2.py3-none-any.whl", - MetadataKind.Sha256, - metadata_name="Requires_Simple.Extra", - ), - ], -} +@pytest.fixture(scope="function") +def download_server_html_index( + script: PipTestEnvironment, + tmpdir: Path, + html_index_with_onetime_server: http.server.ThreadingHTTPServer, +) -> Callable[..., Tuple[TestPipResult, Path]]: + """Execute `pip download` against a generated PyPI index.""" + download_dir = tmpdir / "download_dir" + + def run_for_generated_index( + args: List[str], + allow_error: bool = False, + ) -> Tuple[TestPipResult, Path]: + """ + Produce a PyPI directory structure pointing to the specified packages, then + execute `pip download -i ...` pointing to our generated index. + """ + pip_args = [ + "download", + "-d", + str(download_dir), + "-i", + "http://localhost:8000", + *args, + ] + result = script.pip(*pip_args, allow_error=allow_error) + return (result, download_dir) + + return run_for_generated_index @pytest.mark.parametrize( @@ -1530,14 +1312,13 @@ _simple_packages: Dict[str, List[Package]] = { ], ) def test_download_metadata( - download_generated_html_index: Callable[..., Tuple[TestPipResult, Path]], + download_local_html_index: Callable[..., Tuple[TestPipResult, Path]], requirement_to_download: str, expected_outputs: List[str], ) -> None: """Verify that if a data-dist-info-metadata attribute is present, then it is used instead of the actual dist's METADATA.""" - _, download_dir = download_generated_html_index( - _simple_packages, + _, download_dir = download_local_html_index( [requirement_to_download], ) assert sorted(os.listdir(download_dir)) == expected_outputs @@ -1557,14 +1338,13 @@ def test_download_metadata( ], ) def test_incorrect_metadata_hash( - download_generated_html_index: Callable[..., Tuple[TestPipResult, Path]], + download_local_html_index: Callable[..., Tuple[TestPipResult, Path]], requirement_to_download: str, real_hash: str, ) -> None: """Verify that if a hash for data-dist-info-metadata is provided, it must match the actual hash of the metadata file.""" - result, _ = download_generated_html_index( - _simple_packages, + result, _ = download_local_html_index( [requirement_to_download], allow_error=True, ) @@ -1583,15 +1363,14 @@ def test_incorrect_metadata_hash( ], ) def test_metadata_not_found( - download_generated_html_index: Callable[..., Tuple[TestPipResult, Path]], + download_local_html_index: Callable[..., Tuple[TestPipResult, Path]], requirement_to_download: str, expected_url: str, ) -> None: """Verify that if a data-dist-info-metadata attribute is provided, that pip will fetch the .metadata file at the location specified by PEP 658, and error if unavailable.""" - result, _ = download_generated_html_index( - _simple_packages, + result, _ = download_local_html_index( [requirement_to_download], allow_error=True, ) @@ -1604,11 +1383,10 @@ def test_metadata_not_found( def test_produces_error_for_mismatched_package_name_in_metadata( - download_generated_html_index: Callable[..., Tuple[TestPipResult, Path]], + download_local_html_index: Callable[..., Tuple[TestPipResult, Path]], ) -> None: """Verify that the package name from the metadata matches the requested package.""" - result, _ = download_generated_html_index( - _simple_packages, + result, _ = download_local_html_index( ["simple2==3.0"], allow_error=True, ) @@ -1628,7 +1406,7 @@ def test_produces_error_for_mismatched_package_name_in_metadata( ), ) def test_canonicalizes_package_name_before_verifying_metadata( - download_generated_html_index: Callable[..., Tuple[TestPipResult, Path]], + download_local_html_index: Callable[..., Tuple[TestPipResult, Path]], requirement: str, ) -> None: """Verify that the package name from the command line and the package's @@ -1636,8 +1414,7 @@ def test_canonicalizes_package_name_before_verifying_metadata( Regression test for https://github.com/pypa/pip/issues/12038 """ - result, download_dir = download_generated_html_index( - _simple_packages, + result, download_dir = download_local_html_index( [requirement], allow_error=True, ) From 50a2fb4f9fca0427c192608c30f3cf536b4e4ed4 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed, 2 Aug 2023 00:02:04 -0400 Subject: [PATCH 065/156] add mock server to test that each dist is downloaded exactly once --- tests/conftest.py | 48 +++++++++++++++++++++++++++++ tests/functional/test_download.py | 51 +++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index f481e06c8..25581af9a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -992,3 +992,51 @@ def html_index_for_packages( f.write(pkg_index_content) return html_dir + + +class OneTimeDownloadHandler(http.server.SimpleHTTPRequestHandler): + """Serve files from the current directory, but error if a file is downloaded more + than once.""" + + _seen_paths: ClassVar[Set[str]] = set() + + def do_GET(self) -> None: + if self.path in self._seen_paths: + self.send_error( + http.HTTPStatus.NOT_FOUND, + f"File {self.path} not available more than once!", + ) + return + super().do_GET() + if not (self.path.endswith("/") or self.path.endswith(".metadata")): + self._seen_paths.add(self.path) + + +@pytest.fixture(scope="function") +def html_index_with_onetime_server( + html_index_for_packages: Path, +) -> Iterator[http.server.ThreadingHTTPServer]: + """Serve files from a generated pypi index, erroring if a file is downloaded more + than once. + + Provide `-i http://localhost:8000` to pip invocations to point them at this server. + """ + + class InDirectoryServer(http.server.ThreadingHTTPServer): + def finish_request(self, request: Any, client_address: Any) -> None: + self.RequestHandlerClass( + request, client_address, self, directory=str(html_index_for_packages) # type: ignore[call-arg] # noqa: E501 + ) + + class Handler(OneTimeDownloadHandler): + _seen_paths: ClassVar[Set[str]] = set() + + with InDirectoryServer(("", 8000), Handler) as httpd: + server_thread = threading.Thread(target=httpd.serve_forever) + server_thread.start() + + try: + yield httpd + finally: + httpd.shutdown() + server_thread.join() diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index bedadc704..c204f424b 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -1324,6 +1324,57 @@ def test_download_metadata( assert sorted(os.listdir(download_dir)) == expected_outputs +@pytest.mark.parametrize( + "requirement_to_download, expected_outputs, doubled_path", + [ + ( + "simple2==1.0", + ["simple-1.0.tar.gz", "simple2-1.0.tar.gz"], + "/simple2/simple2-1.0.tar.gz", + ), + ("simple==2.0", ["simple-2.0.tar.gz"], "/simple/simple-2.0.tar.gz"), + ( + "colander", + ["colander-0.9.9-py2.py3-none-any.whl", "translationstring-1.1.tar.gz"], + "/colander/colander-0.9.9-py2.py3-none-any.whl", + ), + ( + "compilewheel", + [ + "compilewheel-1.0-py2.py3-none-any.whl", + "simple-1.0.tar.gz", + ], + "/compilewheel/compilewheel-1.0-py2.py3-none-any.whl", + ), + ], +) +def test_download_metadata_server( + download_server_html_index: Callable[..., Tuple[TestPipResult, Path]], + requirement_to_download: str, + expected_outputs: List[str], + doubled_path: str, +) -> None: + """Verify that if a data-dist-info-metadata attribute is present, then it is used + instead of the actual dist's METADATA. + + Additionally, verify that each dist is downloaded exactly once using a mock server. + + This is a regression test for issue https://github.com/pypa/pip/issues/11847. + """ + _, download_dir = download_server_html_index( + [requirement_to_download, "--no-cache-dir"], + ) + assert sorted(os.listdir(download_dir)) == expected_outputs + shutil.rmtree(download_dir) + result, _ = download_server_html_index( + [requirement_to_download, "--no-cache-dir"], + allow_error=True, + ) + assert result.returncode != 0 + expected_msg = f"File {doubled_path} not available more than once!" + assert expected_msg in result.stderr + + @pytest.mark.parametrize( "requirement_to_download, real_hash", [ From 22637722aa7c9da0aaf58be82b6bef16d6efd097 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue, 1 Aug 2023 20:45:26 -0400 Subject: [PATCH 066/156] fix #11847 for sdists --- src/pip/_internal/operations/prepare.py | 50 ++++++++++++++++--------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 8402be01b..81bf48fbb 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -7,7 +7,7 @@ import mimetypes import os import shutil -from typing import Dict, Iterable, List, Optional +from typing import Dict, Iterable, List, Optional, Set from pip._vendor.packaging.utils import canonicalize_name @@ -474,6 +474,8 @@ class RequirementPreparer: assert req.link links_to_fully_download[req.link] = req + reqs_with_newly_unpacked_source_dirs: Set[Link] = set() + batch_download = self._batch_download( links_to_fully_download.keys(), temp_dir, @@ -481,25 +483,35 @@ class RequirementPreparer: for link, (filepath, _) in batch_download: logger.debug("Downloading link %s to %s", link, filepath) req = links_to_fully_download[link] + # Record the downloaded file path so wheel reqs can extract a Distribution + # in .get_dist(). req.local_file_path = filepath - # TODO: This needs fixing for sdists - # This is an emergency fix for #11847, which reports that - # distributions get downloaded twice when metadata is loaded - # from a PEP 658 standalone metadata file. Setting _downloaded - # fixes this for wheels, but breaks the sdist case (tests - # test_download_metadata). As PyPI is currently only serving - # metadata for wheels, this is not an immediate issue. - # Fixing the problem properly looks like it will require a - # complete refactoring of the `prepare_linked_requirements_more` - # logic, and I haven't a clue where to start on that, so for now - # I have fixed the issue *just* for wheels. - if req.is_wheel: - self._downloaded[req.link.url] = filepath + # Record that the file is downloaded so we don't do it again in + # _prepare_linked_requirement(). + self._downloaded[req.link.url] = filepath + + # If this is an sdist, we need to unpack it and set the .source_dir + # immediately after downloading, as _prepare_linked_requirement() assumes + # the req is either not downloaded at all, or both downloaded and + # unpacked. The downloading and unpacking is is typically done with + # unpack_url(), but we separate the downloading and unpacking steps here in + # order to use the BatchDownloader. + if not req.is_wheel: + hashes = self._get_linked_req_hashes(req) + assert filepath == _check_download_dir(req.link, temp_dir, hashes) + self._ensure_link_req_src_dir(req, parallel_builds) + unpack_file(filepath, req.source_dir) + reqs_with_newly_unpacked_source_dirs.add(req.link) # This step is necessary to ensure all lazy wheels are processed # successfully by the 'download', 'wheel', and 'install' commands. for req in partially_downloaded_reqs: - self._prepare_linked_requirement(req, parallel_builds) + self._prepare_linked_requirement( + req, + parallel_builds, + source_dir_exists_already=req.link + in reqs_with_newly_unpacked_source_dirs, + ) def prepare_linked_requirement( self, req: InstallRequirement, parallel_builds: bool = False @@ -570,7 +582,10 @@ class RequirementPreparer: ) def _prepare_linked_requirement( - self, req: InstallRequirement, parallel_builds: bool + self, + req: InstallRequirement, + parallel_builds: bool, + source_dir_exists_already: bool = False, ) -> BaseDistribution: assert req.link link = req.link @@ -602,7 +617,8 @@ class RequirementPreparer: req.link = req.cached_wheel_source_link link = req.link - self._ensure_link_req_src_dir(req, parallel_builds) + if not source_dir_exists_already: + self._ensure_link_req_src_dir(req, parallel_builds) if link.is_existing_dir(): local_file = None From 957ad95c7d2ca70077a61a3adb551824818f929b Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue, 1 Aug 2023 21:18:15 -0400 Subject: [PATCH 067/156] add news entry --- news/12191.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12191.bugfix.rst diff --git a/news/12191.bugfix.rst b/news/12191.bugfix.rst new file mode 100644 index 000000000..1f384835f --- /dev/null +++ b/news/12191.bugfix.rst @@ -0,0 +1 @@ +Prevent downloading sdists twice when PEP 658 metadata is present. From bfa8a5532d45815d2229ba2e2a920fde6bffc800 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu, 3 Aug 2023 05:50:37 -0400 Subject: [PATCH 068/156] clean up duplicated code --- src/pip/_internal/operations/prepare.py | 54 +++++-------------------- src/pip/_internal/req/req_install.py | 29 ++++++++++++- tests/conftest.py | 52 +++++++++++++----------- 3 files changed, 68 insertions(+), 67 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 81bf48fbb..1b32d7eec 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -7,7 +7,8 @@ import mimetypes import os import shutil -from typing import Dict, Iterable, List, Optional, Set +from pathlib import Path +from typing import Dict, Iterable, List, Optional from pip._vendor.packaging.utils import canonicalize_name @@ -20,7 +21,6 @@ from pip._internal.exceptions import ( InstallationError, MetadataInconsistent, NetworkConnectionError, - PreviousBuildDirError, VcsHashUnsupported, ) from pip._internal.index.package_finder import PackageFinder @@ -47,7 +47,6 @@ from pip._internal.utils.misc import ( display_path, hash_file, hide_url, - is_installable_dir, ) from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.unpacking import unpack_file @@ -319,21 +318,7 @@ class RequirementPreparer: autodelete=True, parallel_builds=parallel_builds, ) - - # If a checkout exists, it's unwise to keep going. version - # inconsistencies are logged later, but do not fail the - # installation. - # FIXME: this won't upgrade when there's an existing - # package unpacked in `req.source_dir` - # TODO: this check is now probably dead code - if is_installable_dir(req.source_dir): - raise PreviousBuildDirError( - "pip can't proceed with requirements '{}' due to a" - "pre-existing build directory ({}). This is likely " - "due to a previous installation that failed . pip is " - "being responsible and not assuming it can delete this. " - "Please delete it and try again.".format(req, req.source_dir) - ) + req.ensure_pristine_source_checkout() def _get_linked_req_hashes(self, req: InstallRequirement) -> Hashes: # By the time this is called, the requirement's link should have @@ -474,8 +459,6 @@ class RequirementPreparer: assert req.link links_to_fully_download[req.link] = req - reqs_with_newly_unpacked_source_dirs: Set[Link] = set() - batch_download = self._batch_download( links_to_fully_download.keys(), temp_dir, @@ -490,28 +473,17 @@ class RequirementPreparer: # _prepare_linked_requirement(). self._downloaded[req.link.url] = filepath - # If this is an sdist, we need to unpack it and set the .source_dir - # immediately after downloading, as _prepare_linked_requirement() assumes - # the req is either not downloaded at all, or both downloaded and - # unpacked. The downloading and unpacking is is typically done with - # unpack_url(), but we separate the downloading and unpacking steps here in - # order to use the BatchDownloader. + # If this is an sdist, we need to unpack it after downloading, but the + # .source_dir won't be set up until we are in _prepare_linked_requirement(). + # Add the downloaded archive to the install requirement to unpack after + # preparing the source dir. if not req.is_wheel: - hashes = self._get_linked_req_hashes(req) - assert filepath == _check_download_dir(req.link, temp_dir, hashes) - self._ensure_link_req_src_dir(req, parallel_builds) - unpack_file(filepath, req.source_dir) - reqs_with_newly_unpacked_source_dirs.add(req.link) + req.needs_unpacked_archive(Path(filepath)) # This step is necessary to ensure all lazy wheels are processed # successfully by the 'download', 'wheel', and 'install' commands. for req in partially_downloaded_reqs: - self._prepare_linked_requirement( - req, - parallel_builds, - source_dir_exists_already=req.link - in reqs_with_newly_unpacked_source_dirs, - ) + self._prepare_linked_requirement(req, parallel_builds) def prepare_linked_requirement( self, req: InstallRequirement, parallel_builds: bool = False @@ -582,10 +554,7 @@ class RequirementPreparer: ) def _prepare_linked_requirement( - self, - req: InstallRequirement, - parallel_builds: bool, - source_dir_exists_already: bool = False, + self, req: InstallRequirement, parallel_builds: bool ) -> BaseDistribution: assert req.link link = req.link @@ -617,8 +586,7 @@ class RequirementPreparer: req.link = req.cached_wheel_source_link link = req.link - if not source_dir_exists_already: - self._ensure_link_req_src_dir(req, parallel_builds) + self._ensure_link_req_src_dir(req, parallel_builds) if link.is_existing_dir(): local_file = None diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 542d6c78f..614c6de9c 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -6,6 +6,7 @@ import sys import uuid import zipfile from optparse import Values +from pathlib import Path from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, Union from pip._vendor.packaging.markers import Marker @@ -17,7 +18,7 @@ from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pyproject_hooks import BuildBackendHookCaller from pip._internal.build_env import BuildEnvironment, NoOpBuildEnvironment -from pip._internal.exceptions import InstallationError +from pip._internal.exceptions import InstallationError, PreviousBuildDirError from pip._internal.locations import get_scheme from pip._internal.metadata import ( BaseDistribution, @@ -47,11 +48,13 @@ from pip._internal.utils.misc import ( backup_dir, display_path, hide_url, + is_installable_dir, redact_auth_from_url, ) from pip._internal.utils.packaging import safe_extra from pip._internal.utils.subprocess import runner_with_spinner_message from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds +from pip._internal.utils.unpacking import unpack_file from pip._internal.utils.virtualenv import running_under_virtualenv from pip._internal.vcs import vcs @@ -180,6 +183,9 @@ class InstallRequirement: # This requirement needs more preparation before it can be built self.needs_more_preparation = False + # This requirement needs to be unpacked before it can be installed. + self._archive_source: Optional[Path] = None + def __str__(self) -> str: if self.req: s = str(self.req) @@ -645,6 +651,27 @@ class InstallRequirement: parallel_builds=parallel_builds, ) + def needs_unpacked_archive(self, archive_source: Path) -> None: + assert self._archive_source is None + self._archive_source = archive_source + + def ensure_pristine_source_checkout(self) -> None: + """Ensure the source directory has not yet been built in.""" + assert self.source_dir is not None + if self._archive_source is not None: + unpack_file(str(self._archive_source), self.source_dir) + elif is_installable_dir(self.source_dir): + # If a checkout exists, it's unwise to keep going. + # version inconsistencies are logged later, but do not fail + # the installation. + raise PreviousBuildDirError( + "pip can't proceed with requirements '{}' due to a " + "pre-existing build directory ({}). This is likely " + "due to a previous installation that failed . pip is " + "being responsible and not assuming it can delete this. " + "Please delete it and try again.".format(self, self.source_dir) + ) + # For editable installations def update_editable(self) -> None: if not self.link: diff --git a/tests/conftest.py b/tests/conftest.py index 25581af9a..cd9931c66 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -935,17 +935,21 @@ def html_index_for_packages( pkg_links = "\n".join( f' {pkg}' for pkg in fake_packages.keys() ) - index_html = f"""\ - - - - - Simple index - - -{pkg_links} - -""" + # Output won't be nicely indented because dedent() acts after f-string + # arg insertion. + index_html = dedent( + f"""\ + + + + + Simple index + + + {pkg_links} + + """ + ) # (2) Generate the index.html in a new subdirectory of the temp directory. (html_dir / "index.html").write_text(index_html) @@ -976,18 +980,20 @@ def html_index_for_packages( # write an index.html with the generated download links for each # copied file for this specific package name. download_links_str = "\n".join(download_links) - pkg_index_content = f"""\ - - - - - Links for {pkg} - - -

Links for {pkg}

-{download_links_str} - -""" + pkg_index_content = dedent( + f"""\ + + + + + Links for {pkg} + + +

Links for {pkg}

+ {download_links_str} + + """ + ) with open(pkg_subdir / "index.html", "w") as f: f.write(pkg_index_content) From 39da6e051a30e90b12608ca90e96e554d82fd15f Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon, 14 Aug 2023 07:55:55 -0400 Subject: [PATCH 069/156] use f-string in exception message --- src/pip/_internal/req/req_install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 614c6de9c..8110114ca 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -665,11 +665,11 @@ class InstallRequirement: # version inconsistencies are logged later, but do not fail # the installation. raise PreviousBuildDirError( - "pip can't proceed with requirements '{}' due to a " - "pre-existing build directory ({}). This is likely " + f"pip can't proceed with requirements '{self}' due to a " + f"pre-existing build directory ({self.source_dir}). This is likely " "due to a previous installation that failed . pip is " "being responsible and not assuming it can delete this. " - "Please delete it and try again.".format(self, self.source_dir) + "Please delete it and try again." ) # For editable installations From 361b02bce0a7283bacd1f26bca64e3facd64aecf Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Mon, 14 Aug 2023 19:09:50 +0200 Subject: [PATCH 070/156] Add is_yanked to installation report --- news/12224.feature.rst | 1 + .../_internal/models/installation_report.py | 3 ++ tests/functional/test_install_report.py | 53 +++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 news/12224.feature.rst diff --git a/news/12224.feature.rst b/news/12224.feature.rst new file mode 100644 index 000000000..5a6977254 --- /dev/null +++ b/news/12224.feature.rst @@ -0,0 +1 @@ +Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but still was selected by pip conform PEP 592. diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index 7f001f35e..31c206751 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -23,6 +23,9 @@ class InstallationReport: # includes editable requirements), and false if the requirement was # downloaded from a PEP 503 index or --find-links. "is_direct": ireq.is_direct, + # is_yanked is true if the requirement was yanked from the index, but + # still was selected by pip conform PEP 592 + "is_yanked": ireq.link.is_yanked if ireq.link else False, # requested is true if the requirement was specified by the user (aka # top level requirement), and false if it was installed as a dependency of a # requirement. https://peps.python.org/pep-0376/#requested diff --git a/tests/functional/test_install_report.py b/tests/functional/test_install_report.py index 003b29d38..a0f855978 100644 --- a/tests/functional/test_install_report.py +++ b/tests/functional/test_install_report.py @@ -64,6 +64,59 @@ def test_install_report_dep( assert _install_dict(report)["simple"]["requested"] is False +def test_yanked_version( + script: PipTestEnvironment, data: TestData, tmp_path: Path +) -> None: + """ + Test is_yanked is True when explicitly requesting a yanked package. + Yanked files are always ignored, unless they are the only file that + matches a version specifier that "pins" to an exact version (PEP 592). + """ + report_path = tmp_path / "report.json" + script.pip( + "install", + "simple==3.0", + "--index-url", + data.index_url("yanked"), + "--dry-run", + "--report", + str(report_path), + allow_stderr_warning=True, + ) + report = json.loads(report_path.read_text()) + simple_report = _install_dict(report)["simple"] + assert simple_report["requested"] is True + assert simple_report["is_direct"] is False + assert simple_report["is_yanked"] is True + assert simple_report["metadata"]["version"] == "3.0" + + +def test_skipped_yanked_version( + script: PipTestEnvironment, data: TestData, tmp_path: Path +) -> None: + """ + Test is_yanked is False when not explicitly requesting a yanked package. + Yanked files are always ignored, unless they are the only file that + matches a version specifier that "pins" to an exact version (PEP 592). + """ + report_path = tmp_path / "report.json" + script.pip( + "install", + "simple", + "--index-url", + data.index_url("yanked"), + "--dry-run", + "--report", + str(report_path), + ) + report = json.loads(report_path.read_text()) + simple_report = _install_dict(report)["simple"] + assert simple_report["requested"] is True + assert simple_report["is_direct"] is False + assert simple_report["is_yanked"] is False + assert simple_report["metadata"]["version"] == "2.0" + + @pytest.mark.network def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> None: """Test report for sdist obtained from index.""" From 553690b39ecb405fd3fb1504d82161e71b02da40 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 15 Aug 2023 10:49:49 +0800 Subject: [PATCH 071/156] Period --- src/pip/_internal/models/installation_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index 31c206751..2acc10d1a 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -24,7 +24,7 @@ class InstallationReport: # downloaded from a PEP 503 index or --find-links. "is_direct": ireq.is_direct, # is_yanked is true if the requirement was yanked from the index, but - # still was selected by pip conform PEP 592 + # still was selected by pip conform PEP 592. "is_yanked": ireq.link.is_yanked if ireq.link else False, # requested is true if the requirement was specified by the user (aka # top level requirement), and false if it was installed as a dependency of a From 3c5e2aed045d68ab49cf2e47bda2826659bc91cc Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Tue, 15 Aug 2023 12:06:56 +0200 Subject: [PATCH 072/156] PR Suggestion Co-authored-by: Paul Moore --- src/pip/_internal/models/installation_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index 2acc10d1a..e38e8f1c0 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -24,7 +24,7 @@ class InstallationReport: # downloaded from a PEP 503 index or --find-links. "is_direct": ireq.is_direct, # is_yanked is true if the requirement was yanked from the index, but - # still was selected by pip conform PEP 592. + # was still selected by pip to conform to PEP 592. "is_yanked": ireq.link.is_yanked if ireq.link else False, # requested is true if the requirement was specified by the user (aka # top level requirement), and false if it was installed as a dependency of a From b4437789a0ed10d8a6c4d76710d512d42a9999ad Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 23 Aug 2023 11:24:51 +0800 Subject: [PATCH 073/156] Fix rtd config --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index b6453d8f0..c0d2bba55 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,7 +6,7 @@ build: python: "3.11" sphinx: - builder: htmldir + builder: dirhtml configuration: docs/html/conf.py python: From 695b9f5ab575cb6043a61d88309a0be038168d0b Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 23 Aug 2023 11:28:18 +0800 Subject: [PATCH 074/156] Upgrade Sphinx to 7.x --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index ef72c8fb7..debfa632b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx ~= 6.0 +sphinx ~= 7.0 towncrier furo myst_parser From 55205b940d451c517f1e66279a6d5a98dd00d275 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 25 Aug 2023 09:43:14 +0200 Subject: [PATCH 075/156] Update installation report docs --- docs/html/reference/installation-report.md | 5 +++++ news/12224.feature.rst | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/html/reference/installation-report.md b/docs/html/reference/installation-report.md index 5823205f9..e0cfcd97e 100644 --- a/docs/html/reference/installation-report.md +++ b/docs/html/reference/installation-report.md @@ -56,6 +56,9 @@ package with the following properties: URL reference. `false` if the requirements was provided as a name and version specifier. +- `is_yanked`: `true` if the requirement was yanked from the index, but was still + selected by pip conform to [PEP 592](https://peps.python.org/pep-0592/#installers). + - `download_info`: Information about the artifact (to be) downloaded for installation, using the [direct URL data structure](https://packaging.python.org/en/latest/specifications/direct-url-data-structure/). @@ -106,6 +109,7 @@ will produce an output similar to this (metadata abriged for brevity): } }, "is_direct": false, + "is_yanked": false, "requested": true, "metadata": { "name": "pydantic", @@ -133,6 +137,7 @@ will produce an output similar to this (metadata abriged for brevity): } }, "is_direct": true, + "is_yanked": false, "requested": true, "metadata": { "name": "packaging", diff --git a/news/12224.feature.rst b/news/12224.feature.rst index 5a6977254..d87426578 100644 --- a/news/12224.feature.rst +++ b/news/12224.feature.rst @@ -1 +1 @@ -Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but still was selected by pip conform PEP 592. +Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to PEP 592. From 510c6acf69cc21f62a10d1c149609890e74bf430 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Sat, 26 Aug 2023 12:17:40 +0200 Subject: [PATCH 076/156] Filter out yanked links from available versions error message --- news/12225.bugfix.rst | 1 + .../resolution/resolvelib/factory.py | 20 +++++++++++++- tests/functional/test_install.py | 27 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 news/12225.bugfix.rst diff --git a/news/12225.bugfix.rst b/news/12225.bugfix.rst new file mode 100644 index 000000000..e1e0c323d --- /dev/null +++ b/news/12225.bugfix.rst @@ -0,0 +1 @@ +Filter out yanked links from the available versions error message: "(from versions: 1.0, 2.0, 3.0)" will not contain yanked versions conform PEP 592. The yanked versions (if any) will be mentioned in a separate error message. diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index ed78580ab..2eb80d4d5 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -603,8 +603,26 @@ class Factory: cands = self._finder.find_all_candidates(req.project_name) skipped_by_requires_python = self._finder.requires_python_skipped_reasons() - versions = [str(v) for v in sorted({c.version for c in cands})] + versions_set: Set[CandidateVersion] = set() + yanked_versions_set: Set[CandidateVersion] = set() + for c in cands: + is_yanked = c.link.is_yanked if c.link else False + if is_yanked: + yanked_versions_set.add(c.version) + else: + versions_set.add(c.version) + + versions = [str(v) for v in sorted(versions_set)] + yanked_versions = [str(v) for v in sorted(yanked_versions_set)] + + if yanked_versions: + # Saying "version X is yanked" isn't entirely accurate. + # https://github.com/pypa/pip/issues/11745#issuecomment-1402805842 + logger.critical( + "Ignored the following yanked versions: %s", + ", ".join(yanked_versions) or "none", + ) if skipped_by_requires_python: logger.critical( "Ignored the following versions that require a different python " diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 5e8a82fb3..161881419 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -2242,6 +2242,33 @@ def test_install_yanked_file_and_print_warning( assert "Successfully installed simple-3.0\n" in result.stdout, str(result) +def test_yanked_version_missing_from_availble_versions_error_message( + script: PipTestEnvironment, data: TestData +) -> None: + """ + Test yanked version is missing from available versions error message. + + Yanked files are always ignored, unless they are the only file that + matches a version specifier that "pins" to an exact version (PEP 592). + """ + result = script.pip( + "install", + "simple==", + "--index-url", + data.index_url("yanked"), + expect_error=True, + ) + # the yanked version (3.0) is filtered out from the output: + expected_warning = ( + "Could not find a version that satisfies the requirement simple== " + "(from versions: 1.0, 2.0)" + ) + assert expected_warning in result.stderr, str(result) + # and mentioned in a separate warning: + expected_warning = "Ignored the following yanked versions: 3.0" + assert expected_warning in result.stderr, str(result) + + def test_error_all_yanked_files_and_no_pin( script: PipTestEnvironment, data: TestData ) -> None: From 69a1e956dae1d6ced4bc6e66883b271f1f7a10e9 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 26 Aug 2023 10:20:40 +0200 Subject: [PATCH 077/156] Ruff rules C4,C90,PERF --- .pre-commit-config.yaml | 2 +- ...FF-ABE1-48C7-954C-7C3EB229135F.feature.rst | 1 + pyproject.toml | 30 +++++++++++++---- src/pip/_internal/cache.py | 6 ++-- src/pip/_internal/cli/autocompletion.py | 5 +-- src/pip/_internal/commands/cache.py | 12 ++----- src/pip/_internal/commands/debug.py | 2 +- src/pip/_internal/commands/list.py | 2 +- src/pip/_internal/locations/_distutils.py | 2 +- .../_internal/models/installation_report.py | 2 +- src/pip/_internal/operations/install/wheel.py | 6 ++-- src/pip/_internal/req/req_uninstall.py | 2 +- tests/functional/test_cache.py | 20 ++++-------- tests/functional/test_help.py | 4 +-- tests/functional/test_list.py | 32 ++++++++----------- tests/lib/__init__.py | 18 +++++------ tests/unit/test_finder.py | 6 ++-- tests/unit/test_logging.py | 14 ++++---- tests/unit/test_req_uninstall.py | 7 ++-- tests/unit/test_self_check_outdated.py | 2 +- tests/unit/test_target_python.py | 16 +++++----- tests/unit/test_vcs.py | 2 +- 22 files changed, 93 insertions(+), 100 deletions(-) create mode 100644 news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.feature.rst diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0aef0d60..1c497c294 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.0.270 + rev: v0.0.286 hooks: - id: ruff diff --git a/news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.feature.rst b/news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.feature.rst new file mode 100644 index 000000000..7f6c1d561 --- /dev/null +++ b/news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.feature.rst @@ -0,0 +1 @@ +Add ruff rules ASYNC,C4,C90,PERF,PLE,PLR for minor optimizations and to set upper limits on code complexity. diff --git a/pyproject.toml b/pyproject.toml index b7c0d1545..c3c21802f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,9 +74,9 @@ webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LIC [tool.ruff] extend-exclude = [ + "_vendor", "./build", ".scratch", - "_vendor", "data", ] ignore = [ @@ -88,21 +88,37 @@ ignore = [ ] line-length = 88 select = [ + "ASYNC", "B", + "C4", + "C90", "E", "F", - "W", "G", - "ISC", "I", + "ISC", + "PERF", + "PLE", + "PLR0", + "W", ] -[tool.ruff.per-file-ignores] -"noxfile.py" = ["G"] -"tests/*" = ["B011"] - [tool.ruff.isort] # We need to explicitly make pip "first party" as it's imported by code in # the docs and tests directories. known-first-party = ["pip"] known-third-party = ["pip._vendor"] + +[tool.ruff.mccabe] +max-complexity = 33 # default is 10 + +[tool.ruff.per-file-ignores] +"noxfile.py" = ["G"] +"src/pip/_internal/*" = ["PERF203"] +"tests/*" = ["B011"] + +[tool.ruff.pylint] +max-args = 15 # default is 5 +max-branches = 28 # default is 12 +max-returns = 13 # default is 6 +max-statements = 134 # default is 50 diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 8d3a664c7..f45ac23e9 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -78,12 +78,10 @@ class Cache: if can_not_cache: return [] - candidates = [] path = self.get_path_for_link(link) if os.path.isdir(path): - for candidate in os.listdir(path): - candidates.append((candidate, path)) - return candidates + return [(candidate, path) for candidate in os.listdir(path)] + return [] def get_path_for_link(self, link: Link) -> str: """Return a directory to store cached items in for link.""" diff --git a/src/pip/_internal/cli/autocompletion.py b/src/pip/_internal/cli/autocompletion.py index 226fe84dc..e5950b906 100644 --- a/src/pip/_internal/cli/autocompletion.py +++ b/src/pip/_internal/cli/autocompletion.py @@ -71,8 +71,9 @@ def autocomplete() -> None: for opt in subcommand.parser.option_list_all: if opt.help != optparse.SUPPRESS_HELP: - for opt_str in opt._long_opts + opt._short_opts: - options.append((opt_str, opt.nargs)) + options += [ + (opt_str, opt.nargs) for opt_str in opt._long_opts + opt._short_opts + ] # filter out previously specified options from available options prev_opts = [x.split("=")[0] for x in cwords[1 : cword - 1]] diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index e96d2b492..f6430980c 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -3,10 +3,10 @@ import textwrap from optparse import Values from typing import Any, List -import pip._internal.utils.filesystem as filesystem from pip._internal.cli.base_command import Command from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.exceptions import CommandError, PipError +from pip._internal.utils import filesystem from pip._internal.utils.logging import getLogger logger = getLogger(__name__) @@ -151,14 +151,8 @@ class CacheCommand(Command): logger.info("\n".join(sorted(results))) def format_for_abspath(self, files: List[str]) -> None: - if not files: - return - - results = [] - for filename in files: - results.append(filename) - - logger.info("\n".join(sorted(results))) + if files: + logger.info("\n".join(sorted(files))) def remove_cache_items(self, options: Values, args: List[Any]) -> None: if len(args) > 1: diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index 88a4f798d..564409c68 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -135,7 +135,7 @@ def show_tags(options: Values) -> None: def ca_bundle_info(config: Configuration) -> str: levels = set() - for key, _ in config.items(): + for key, _ in config.items(): # noqa: PERF102 Configuration has no keys() method. levels.add(key.split(".")[0]) if not levels: diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index ac1035319..2ec456b95 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -297,7 +297,7 @@ class ListCommand(IndexGroupCommand): # Create and add a separator. if len(data) > 0: - pkg_strings.insert(1, " ".join(map(lambda x: "-" * x, sizes))) + pkg_strings.insert(1, " ".join(("-" * x for x in sizes))) for val in pkg_strings: write_output(val) diff --git a/src/pip/_internal/locations/_distutils.py b/src/pip/_internal/locations/_distutils.py index 92bd93179..48689f5fb 100644 --- a/src/pip/_internal/locations/_distutils.py +++ b/src/pip/_internal/locations/_distutils.py @@ -89,7 +89,7 @@ def distutils_scheme( # finalize_options(); we only want to override here if the user # has explicitly requested it hence going back to the config if "install_lib" in d.get_option_dict("install"): - scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib)) + scheme.update({"purelib": i.install_lib, "platlib": i.install_lib}) if running_under_virtualenv(): if home: diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index 7f001f35e..da0334bd5 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -33,7 +33,7 @@ class InstallationReport: } if ireq.user_supplied and ireq.extras: # For top level requirements, the list of requested extras, if any. - res["requested_extras"] = list(sorted(ireq.extras)) + res["requested_extras"] = sorted(ireq.extras) return res def to_dict(self) -> Dict[str, Any]: diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index a8cd1330f..58a773059 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -267,9 +267,9 @@ def get_csv_rows_for_installed( path = _fs_to_record_path(f, lib_dir) digest, length = rehash(f) installed_rows.append((path, digest, length)) - for installed_record_path in installed.values(): - installed_rows.append((installed_record_path, "", "")) - return installed_rows + return installed_rows + [ + (installed_record_path, "", "") for installed_record_path in installed.values() + ] def get_console_script_specs(console: Dict[str, str]) -> List[str]: diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index ad5178e76..861aa4f22 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -274,7 +274,7 @@ class StashedUninstallPathSet: def commit(self) -> None: """Commits the uninstall by removing stashed files.""" - for _, save_dir in self._save_dirs.items(): + for save_dir in self._save_dirs.values(): save_dir.cleanup() self._moves = [] self._save_dirs = {} diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index 788abdd2b..a6640cbbf 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -36,10 +36,7 @@ def http_cache_files(http_cache_dir: str) -> List[str]: return [] filenames = glob(os.path.join(destination, "*")) - files = [] - for filename in filenames: - files.append(os.path.join(destination, filename)) - return files + return [os.path.join(destination, filename) for filename in filenames] @pytest.fixture @@ -50,10 +47,7 @@ def wheel_cache_files(wheel_cache_dir: str) -> List[str]: return [] filenames = glob(os.path.join(destination, "*.whl")) - files = [] - for filename in filenames: - files.append(os.path.join(destination, filename)) - return files + return [os.path.join(destination, filename) for filename in filenames] @pytest.fixture @@ -107,7 +101,7 @@ def list_matches_wheel(wheel_name: str, result: TestPipResult) -> bool: `- foo-1.2.3-py3-none-any.whl `.""" lines = result.stdout.splitlines() expected = f" - {wheel_name}-py3-none-any.whl " - return any(map(lambda line: line.startswith(expected), lines)) + return any((line.startswith(expected) for line in lines)) def list_matches_wheel_abspath(wheel_name: str, result: TestPipResult) -> bool: @@ -120,11 +114,9 @@ def list_matches_wheel_abspath(wheel_name: str, result: TestPipResult) -> bool: lines = result.stdout.splitlines() expected = f"{wheel_name}-py3-none-any.whl" return any( - map( - lambda line: ( - os.path.basename(line).startswith(expected) and os.path.exists(line) - ), - lines, + ( + (os.path.basename(line).startswith(expected) and os.path.exists(line)) + for line in lines ) ) diff --git a/tests/functional/test_help.py b/tests/functional/test_help.py index dba41af5f..9627a1215 100644 --- a/tests/functional/test_help.py +++ b/tests/functional/test_help.py @@ -102,8 +102,8 @@ def test_help_commands_equally_functional(in_memory_pip: InMemoryPip) -> None: results = list(map(in_memory_pip.pip, ("help", "--help"))) results.append(in_memory_pip.pip()) - out = map(lambda x: x.stdout, results) - ret = map(lambda x: x.returncode, results) + out = (x.stdout for x in results) + ret = (x.returncode for x in results) msg = '"pip --help" != "pip help" != "pip"' assert len(set(out)) == 1, "output of: " + msg diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index 4f2be8387..cf8900a32 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -273,25 +273,19 @@ def test_outdated_flag(script: PipTestEnvironment, data: TestData) -> None: "latest_version": "3.0", "latest_filetype": "sdist", } in json_output - assert ( - dict( - name="simplewheel", - version="1.0", - latest_version="2.0", - latest_filetype="wheel", - ) - in json_output - ) - assert ( - dict( - name="pip-test-package", - version="0.1", - latest_version="0.1.1", - latest_filetype="sdist", - editable_project_location="", - ) - in json_output - ) + assert { + "name": "simplewheel", + "version": "1.0", + "latest_version": "2.0", + "latest_filetype": "wheel", + } in json_output + assert { + "name": "pip-test-package", + "version": "0.1", + "latest_version": "0.1.1", + "latest_filetype": "sdist", + "editable_project_location": "", + } in json_output assert "simple2" not in {p["name"] for p in json_output} diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index b827f88ba..3c8ca98f7 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -740,21 +740,19 @@ class PipTestEnvironment(TestFileEnvironment): def assert_installed(self, **kwargs: str) -> None: ret = self.pip("list", "--format=json") - installed = set( + installed = { (canonicalize_name(val["name"]), val["version"]) for val in json.loads(ret.stdout) - ) - expected = set((canonicalize_name(k), v) for k, v in kwargs.items()) + } + expected = {(canonicalize_name(k), v) for k, v in kwargs.items()} assert expected <= installed, "{!r} not all in {!r}".format(expected, installed) def assert_not_installed(self, *args: str) -> None: ret = self.pip("list", "--format=json") - installed = set( - canonicalize_name(val["name"]) for val in json.loads(ret.stdout) - ) + installed = {canonicalize_name(val["name"]) for val in json.loads(ret.stdout)} # None of the given names should be listed as installed, i.e. their # intersection should be empty. - expected = set(canonicalize_name(k) for k in args) + expected = {canonicalize_name(k) for k in args} assert not (expected & installed), "{!r} contained in {!r}".format( expected, installed ) @@ -798,16 +796,16 @@ def diff_states( return path.startswith(prefix) start_keys = { - k for k in start.keys() if not any([prefix_match(k, i) for i in ignore]) + k for k in start.keys() if not any(prefix_match(k, i) for i in ignore) } - end_keys = {k for k in end.keys() if not any([prefix_match(k, i) for i in ignore])} + end_keys = {k for k in end.keys() if not any(prefix_match(k, i) for i in ignore)} deleted = {k: start[k] for k in start_keys.difference(end_keys)} created = {k: end[k] for k in end_keys.difference(start_keys)} updated = {} for k in start_keys.intersection(end_keys): if start[k].size != end[k].size: updated[k] = end[k] - return dict(deleted=deleted, created=created, updated=updated) + return {"deleted": deleted, "created": created, "updated": updated} def assert_all_changes( diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 3404d1498..393e83d5a 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -234,7 +234,7 @@ class TestCandidateEvaluator: ) sort_key = evaluator._sort_key results = sorted(links, key=sort_key, reverse=True) - results2 = sorted(reversed(links), key=sort_key, reverse=True) + results2 = sorted(links, key=sort_key, reverse=True) assert links == results, results assert links == results2, results2 @@ -261,7 +261,7 @@ class TestCandidateEvaluator: candidate_evaluator = CandidateEvaluator.create("my-project") sort_key = candidate_evaluator._sort_key results = sorted(links, key=sort_key, reverse=True) - results2 = sorted(reversed(links), key=sort_key, reverse=True) + results2 = sorted(links, key=sort_key, reverse=True) assert links == results, results assert links == results2, results2 @@ -301,7 +301,7 @@ class TestCandidateEvaluator: ) sort_key = evaluator._sort_key results = sorted(links, key=sort_key, reverse=True) - results2 = sorted(reversed(links), key=sort_key, reverse=True) + results2 = sorted(links, key=sort_key, reverse=True) assert links == results, results assert links == results2, results2 diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index 3ba6ed57c..9d507d742 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -21,13 +21,13 @@ class TestIndentingFormatter: def make_record(self, msg: str, level_name: str) -> logging.LogRecord: level_number = getattr(logging, level_name) - attrs = dict( - msg=msg, - created=1547704837.040001 + time.timezone, - msecs=40, - levelname=level_name, - levelno=level_number, - ) + attrs = { + "msg": msg, + "created": 1547704837.040001 + time.timezone, + "msecs": 40, + "levelname": level_name, + "levelno": level_number, + } record = logging.makeLogRecord(attrs) return record diff --git a/tests/unit/test_req_uninstall.py b/tests/unit/test_req_uninstall.py index b4ae97350..6a846e202 100644 --- a/tests/unit/test_req_uninstall.py +++ b/tests/unit/test_req_uninstall.py @@ -59,10 +59,9 @@ def test_uninstallation_paths() -> None: def test_compressed_listing(tmpdir: Path) -> None: def in_tmpdir(paths: List[str]) -> List[str]: - li = [] - for path in paths: - li.append(str(os.path.join(tmpdir, path.replace("/", os.path.sep)))) - return li + return [ + str(os.path.join(tmpdir, path.replace("/", os.path.sep))) for path in paths + ] sample = in_tmpdir( [ diff --git a/tests/unit/test_self_check_outdated.py b/tests/unit/test_self_check_outdated.py index c025ff302..6b2333f18 100644 --- a/tests/unit/test_self_check_outdated.py +++ b/tests/unit/test_self_check_outdated.py @@ -40,7 +40,7 @@ def test_pip_self_version_check_calls_underlying_implementation( ) -> None: # GIVEN mock_session = Mock() - fake_options = Values(dict(cache_dir=str(tmpdir))) + fake_options = Values({"cache_dir": str(tmpdir)}) # WHEN self_outdated_check.pip_self_version_check(mock_session, fake_options) diff --git a/tests/unit/test_target_python.py b/tests/unit/test_target_python.py index b659c61fe..bc1713769 100644 --- a/tests/unit/test_target_python.py +++ b/tests/unit/test_target_python.py @@ -54,18 +54,18 @@ class TestTargetPython: "kwargs, expected", [ ({}, ""), - (dict(py_version_info=(3, 6)), "version_info='3.6'"), + ({"py_version_info": (3, 6)}, "version_info='3.6'"), ( - dict(platforms=["darwin"], py_version_info=(3, 6)), + {"platforms": ["darwin"], "py_version_info": (3, 6)}, "platforms=['darwin'] version_info='3.6'", ), ( - dict( - platforms=["darwin"], - py_version_info=(3, 6), - abis=["cp36m"], - implementation="cp", - ), + { + "platforms": ["darwin"], + "py_version_info": (3, 6), + "abis": ["cp36m"], + "implementation": "cp", + }, ( "platforms=['darwin'] version_info='3.6' abis=['cp36m'] " "implementation='cp'" diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 38daaa0f2..3ecc69abf 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -73,7 +73,7 @@ def test_rev_options_repr() -> None: Git, ["HEAD", "opt1", "opt2"], ["123", "opt1", "opt2"], - dict(extra_args=["opt1", "opt2"]), + {"extra_args": ["opt1", "opt2"]}, ), ], ) From 0a24a001fbe451aa399063555634f8d971776a21 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 28 Aug 2023 15:04:54 +0200 Subject: [PATCH 078/156] Fix issues raised in code review --- ...rst => 4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst} | 0 pyproject.toml | 1 + src/pip/_internal/commands/debug.py | 5 +---- src/pip/_internal/commands/list.py | 2 +- tests/functional/test_cache.py | 2 +- tests/lib/__init__.py | 6 ++---- tests/unit/test_finder.py | 6 +++--- 7 files changed, 9 insertions(+), 13 deletions(-) rename news/{4A0C40FF-ABE1-48C7-954C-7C3EB229135F.feature.rst => 4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst} (100%) diff --git a/news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.feature.rst b/news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst similarity index 100% rename from news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.feature.rst rename to news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst diff --git a/pyproject.toml b/pyproject.toml index c3c21802f..7a4fe6246 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ max-complexity = 33 # default is 10 "noxfile.py" = ["G"] "src/pip/_internal/*" = ["PERF203"] "tests/*" = ["B011"] +"tests/unit/test_finder.py" = ["C414"] [tool.ruff.pylint] max-args = 15 # default is 5 diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index 564409c68..3d6416023 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -134,10 +134,7 @@ def show_tags(options: Values) -> None: def ca_bundle_info(config: Configuration) -> str: - levels = set() - for key, _ in config.items(): # noqa: PERF102 Configuration has no keys() method. - levels.add(key.split(".")[0]) - + levels = {key.split(".")[0] for key, _ in config.items()} # noqa: PERF102 if not levels: return "Not specified" diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 2ec456b95..e551dda9a 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -297,7 +297,7 @@ class ListCommand(IndexGroupCommand): # Create and add a separator. if len(data) > 0: - pkg_strings.insert(1, " ".join(("-" * x for x in sizes))) + pkg_strings.insert(1, " ".join("-" * x for x in sizes)) for val in pkg_strings: write_output(val) diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index a6640cbbf..8bee7e4fc 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -101,7 +101,7 @@ def list_matches_wheel(wheel_name: str, result: TestPipResult) -> bool: `- foo-1.2.3-py3-none-any.whl `.""" lines = result.stdout.splitlines() expected = f" - {wheel_name}-py3-none-any.whl " - return any((line.startswith(expected) for line in lines)) + return any(line.startswith(expected) for line in lines) def list_matches_wheel_abspath(wheel_name: str, result: TestPipResult) -> bool: diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 3c8ca98f7..a48423570 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -795,10 +795,8 @@ def diff_states( prefix = prefix.rstrip(os.path.sep) + os.path.sep return path.startswith(prefix) - start_keys = { - k for k in start.keys() if not any(prefix_match(k, i) for i in ignore) - } - end_keys = {k for k in end.keys() if not any(prefix_match(k, i) for i in ignore)} + start_keys = {k for k in start if not any(prefix_match(k, i) for i in ignore)} + end_keys = {k for k in end if not any(prefix_match(k, i) for i in ignore)} deleted = {k: start[k] for k in start_keys.difference(end_keys)} created = {k: end[k] for k in end_keys.difference(start_keys)} updated = {} diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 393e83d5a..3404d1498 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -234,7 +234,7 @@ class TestCandidateEvaluator: ) sort_key = evaluator._sort_key results = sorted(links, key=sort_key, reverse=True) - results2 = sorted(links, key=sort_key, reverse=True) + results2 = sorted(reversed(links), key=sort_key, reverse=True) assert links == results, results assert links == results2, results2 @@ -261,7 +261,7 @@ class TestCandidateEvaluator: candidate_evaluator = CandidateEvaluator.create("my-project") sort_key = candidate_evaluator._sort_key results = sorted(links, key=sort_key, reverse=True) - results2 = sorted(links, key=sort_key, reverse=True) + results2 = sorted(reversed(links), key=sort_key, reverse=True) assert links == results, results assert links == results2, results2 @@ -301,7 +301,7 @@ class TestCandidateEvaluator: ) sort_key = evaluator._sort_key results = sorted(links, key=sort_key, reverse=True) - results2 = sorted(links, key=sort_key, reverse=True) + results2 = sorted(reversed(links), key=sort_key, reverse=True) assert links == results, results assert links == results2, results2 From c127512f13f933a15123f90489e36192060889a5 Mon Sep 17 00:00:00 2001 From: studioj <22102283+studioj@users.noreply.github.com> Date: Thu, 24 Aug 2023 23:23:46 +0200 Subject: [PATCH 079/156] small update for authentication.md --- docs/html/topics/authentication.md | 2 +- news/zhsdgdlsjgksdfj.trivial.rst | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 news/zhsdgdlsjgksdfj.trivial.rst diff --git a/docs/html/topics/authentication.md b/docs/html/topics/authentication.md index 966ac3e7a..a26490717 100644 --- a/docs/html/topics/authentication.md +++ b/docs/html/topics/authentication.md @@ -68,7 +68,7 @@ man pages][netrc-docs]. pip supports loading credentials stored in your keyring using the {pypi}`keyring` library, which can be enabled py passing `--keyring-provider` with a value of `auto`, `disabled`, `import`, or `subprocess`. The default -value `auto` respects `--no-input` and not query keyring at all if the option +value `auto` respects `--no-input` and does not query keyring at all if the option is used; otherwise it tries the `import`, `subprocess`, and `disabled` providers (in this order) and uses the first one that works. diff --git a/news/zhsdgdlsjgksdfj.trivial.rst b/news/zhsdgdlsjgksdfj.trivial.rst new file mode 100644 index 000000000..e69de29bb From 0c0099b23b1109cd26c66def926ffab4c99f73cd Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 29 Aug 2023 11:31:42 +0200 Subject: [PATCH 080/156] Ruff misidentifies config as a dict Co-authored-by: Tzu-ping Chung --- src/pip/_internal/commands/debug.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index 3d6416023..f76e033df 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -134,6 +134,8 @@ def show_tags(options: Values) -> None: def ca_bundle_info(config: Configuration) -> str: + # Ruff misidentifies config as a dict. + # Configuration does not have support the mapping interface. levels = {key.split(".")[0] for key, _ in config.items()} # noqa: PERF102 if not levels: return "Not specified" From f61250303515e04c8bfc5306d91686cc0d661ee9 Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Thu, 31 Aug 2023 04:28:31 -0400 Subject: [PATCH 081/156] Remove uses of `utcnow` in non-vendored code (#12006) * Remove reference to utcnow This cleans up some of the datetime handling in the self check. Note that this changes the format of the state file, since the datetime now uses ``.isoformat()`` instead of ``.strftime``. Reading an outdated state file will still work on Python 3.11+, but not on earlier versions. * Use aware datetime object in x509.CertificateBuilder --- news/12005.bugfix.rst | 1 + src/pip/_internal/self_outdated_check.py | 15 ++++++--------- tests/lib/certs.py | 6 +++--- tests/unit/test_self_check_outdated.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 15 deletions(-) create mode 100644 news/12005.bugfix.rst diff --git a/news/12005.bugfix.rst b/news/12005.bugfix.rst new file mode 100644 index 000000000..98a3e5112 --- /dev/null +++ b/news/12005.bugfix.rst @@ -0,0 +1 @@ +Removed uses of ``datetime.datetime.utcnow`` from non-vendored code. diff --git a/src/pip/_internal/self_outdated_check.py b/src/pip/_internal/self_outdated_check.py index 41cc42c56..eefbc498b 100644 --- a/src/pip/_internal/self_outdated_check.py +++ b/src/pip/_internal/self_outdated_check.py @@ -28,8 +28,7 @@ from pip._internal.utils.entrypoints import ( from pip._internal.utils.filesystem import adjacent_tmp_file, check_path_owner, replace from pip._internal.utils.misc import ensure_dir -_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" - +_WEEK = datetime.timedelta(days=7) logger = logging.getLogger(__name__) @@ -73,12 +72,10 @@ class SelfCheckState: if "pypi_version" not in self._state: return None - seven_days_in_seconds = 7 * 24 * 60 * 60 - # Determine if we need to refresh the state - last_check = datetime.datetime.strptime(self._state["last_check"], _DATE_FMT) - seconds_since_last_check = (current_time - last_check).total_seconds() - if seconds_since_last_check > seven_days_in_seconds: + last_check = datetime.datetime.fromisoformat(self._state["last_check"]) + time_since_last_check = current_time - last_check + if time_since_last_check > _WEEK: return None return self._state["pypi_version"] @@ -100,7 +97,7 @@ class SelfCheckState: # Include the key so it's easy to tell which pip wrote the # file. "key": self.key, - "last_check": current_time.strftime(_DATE_FMT), + "last_check": current_time.isoformat(), "pypi_version": pypi_version, } @@ -229,7 +226,7 @@ def pip_self_version_check(session: PipSession, options: optparse.Values) -> Non try: upgrade_prompt = _self_version_check_logic( state=SelfCheckState(cache_dir=options.cache_dir), - current_time=datetime.datetime.utcnow(), + current_time=datetime.datetime.now(datetime.timezone.utc), local_version=installed_dist.version, get_remote_version=functools.partial( _get_current_remote_pip_version, session, options diff --git a/tests/lib/certs.py b/tests/lib/certs.py index 54b484ac0..9e6542d2d 100644 --- a/tests/lib/certs.py +++ b/tests/lib/certs.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Tuple from cryptography import x509 @@ -23,8 +23,8 @@ def make_tls_cert(hostname: str) -> Tuple[x509.Certificate, rsa.RSAPrivateKey]: .issuer_name(issuer) .public_key(key.public_key()) .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.utcnow()) - .not_valid_after(datetime.utcnow() + timedelta(days=10)) + .not_valid_before(datetime.now(timezone.utc)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(days=10)) .add_extension( x509.SubjectAlternativeName([x509.DNSName(hostname)]), critical=False, diff --git a/tests/unit/test_self_check_outdated.py b/tests/unit/test_self_check_outdated.py index c025ff302..011df08ae 100644 --- a/tests/unit/test_self_check_outdated.py +++ b/tests/unit/test_self_check_outdated.py @@ -49,7 +49,9 @@ def test_pip_self_version_check_calls_underlying_implementation( mocked_state.assert_called_once_with(cache_dir=str(tmpdir)) mocked_function.assert_called_once_with( state=mocked_state(cache_dir=str(tmpdir)), - current_time=datetime.datetime(1970, 1, 2, 11, 0, 0), + current_time=datetime.datetime( + 1970, 1, 2, 11, 0, 0, tzinfo=datetime.timezone.utc + ), local_version=ANY, get_remote_version=ANY, ) @@ -167,7 +169,10 @@ class TestSelfCheckState: # WHEN state = self_outdated_check.SelfCheckState(cache_dir=str(cache_dir)) - state.set("1.0.0", datetime.datetime(2000, 1, 1, 0, 0, 0)) + state.set( + "1.0.0", + datetime.datetime(2000, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), + ) # THEN assert state._statefile_path == os.fspath(expected_path) @@ -175,6 +180,6 @@ class TestSelfCheckState: contents = expected_path.read_text() assert json.loads(contents) == { "key": sys.prefix, - "last_check": "2000-01-01T00:00:00Z", + "last_check": "2000-01-01T00:00:00+00:00", "pypi_version": "1.0.0", } From 50c49f1d8340980a80f2d5248a66d62d07697358 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 1 Sep 2023 22:41:00 +0200 Subject: [PATCH 082/156] GitHub Actions: setup-python allow-prereleases for 3.12 (#12252) --- .github/workflows/ci.yml | 6 +++--- news/12252.trivial.rst | 0 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 news/12252.trivial.rst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50ec976af..41d3ab946 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,14 +109,14 @@ jobs: - "3.9" - "3.10" - "3.11" - - key: "3.12" - full: "3.12-dev" + - "3.12" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python.full || matrix.python }} + python-version: ${{ matrix.python }} + allow-prereleases: true - name: Install Ubuntu dependencies if: matrix.os == 'Ubuntu' diff --git a/news/12252.trivial.rst b/news/12252.trivial.rst new file mode 100644 index 000000000..e69de29bb From a88e73b29870d0682049ee88b4d9f6239e5f238b Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 2 Sep 2023 12:12:54 +0200 Subject: [PATCH 083/156] Fix typos in dep resolution notes doc --- docs/html/topics/more-dependency-resolution.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/html/topics/more-dependency-resolution.md b/docs/html/topics/more-dependency-resolution.md index 31967a6a9..1c7836e5c 100644 --- a/docs/html/topics/more-dependency-resolution.md +++ b/docs/html/topics/more-dependency-resolution.md @@ -8,7 +8,7 @@ and this article is intended to help readers understand what is happening ```{note} This document is a work in progress. The details included are accurate (at the time of writing), but there is additional information, in particular around -pip's interface with resolvelib, which have not yet been included. +pip's interface with resolvelib, which has not yet been included. Contributions to improve this document are welcome. ``` @@ -26,7 +26,7 @@ The practical implication of that is that there will always be some situations where pip cannot determine what to install in a reasonable length of time. We make every effort to ensure that such situations happen rarely, but eliminating them altogether isn't even theoretically possible. We'll discuss what options -yopu have if you hit a problem situation like this a little later. +you have if you hit a problem situation like this a little later. ## Python specific issues @@ -136,7 +136,7 @@ operations: that satisfy them. This is essentially where the finder interacts with the resolver. * `is_satisfied_by` - checks if a candidate satisfies a requirement. This is - basically the implementation of what a requirement meams. + basically the implementation of what a requirement means. * `get_dependencies` - get the dependency metadata for a candidate. This is the implementation of the process of getting and reading package metadata. From 7c5b2f2ca9dbb4bc2ff638fe09a11e332fb1123a Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 5 Sep 2023 17:31:55 -0500 Subject: [PATCH 084/156] Update security policy (#12254) Provide a link to the CNA/PSRT disclosure process. --- SECURITY.md | 11 +++++++++-- news/12254.process.rst | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 news/12254.process.rst diff --git a/SECURITY.md b/SECURITY.md index 4e423805a..e75a1c0de 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,10 @@ -# Security and Vulnerability Reporting +# Security Policy -If you find any security issues, please report to [security@python.org](mailto:security@python.org) +## Reporting a Vulnerability + +Please read the guidelines on reporting security issues [on the +official website](https://www.python.org/dev/security/) for +instructions on how to report a security-related problem to +the Python Security Response Team responsibly. + +To reach the response team, email `security at python dot org`. diff --git a/news/12254.process.rst b/news/12254.process.rst new file mode 100644 index 000000000..e54690268 --- /dev/null +++ b/news/12254.process.rst @@ -0,0 +1 @@ +Added reference to `vulnerability reporting guidelines `_ to pip's security policy. From af43e139b626ab67e5bccae5fa1645b1dbb68fec Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 6 Sep 2023 15:39:11 +0800 Subject: [PATCH 085/156] Drive by split() limit to improve performance --- src/pip/_internal/commands/debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index f76e033df..1b1fd3ea5 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -136,7 +136,7 @@ def show_tags(options: Values) -> None: def ca_bundle_info(config: Configuration) -> str: # Ruff misidentifies config as a dict. # Configuration does not have support the mapping interface. - levels = {key.split(".")[0] for key, _ in config.items()} # noqa: PERF102 + levels = {key.split(".", 1)[0] for key, _ in config.items()} # noqa: PERF102 if not levels: return "Not specified" From 99a00f0a8deee30dafb7448eefdbc1c9b3b6062d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 6 Sep 2023 11:36:19 +0200 Subject: [PATCH 086/156] pre-commit autoupdate except mypy --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1c497c294..c8d81deed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,13 +17,13 @@ repos: exclude: .patch - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.0.286 + rev: v0.0.287 hooks: - id: ruff From 2281e91d4e19ece6b279133c4eaf2632fcf204bc Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 6 Sep 2023 11:44:45 +0200 Subject: [PATCH 087/156] pre-commit autoupdate except mypy --- news/12261.trivial.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 news/12261.trivial.rst diff --git a/news/12261.trivial.rst b/news/12261.trivial.rst new file mode 100644 index 000000000..e69de29bb From 21bfe401a96ddecb2a827b9b3cd5ff1b833b151f Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 6 Sep 2023 11:50:10 +0200 Subject: [PATCH 088/156] use more stable sort key --- src/pip/_internal/resolution/resolvelib/resolver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 4c53dfb25..2e4941da8 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -104,8 +104,9 @@ class Resolver(BaseResolver): raise error from e req_set = RequirementSet(check_supported_wheels=check_supported_wheels) - # sort to ensure base candidates come before candidates with extras - for candidate in sorted(result.mapping.values(), key=lambda c: c.name): + # process candidates with extras last to ensure their base equivalent is already in the req_set if appropriate + # Python's sort is stable so using a binary key function keeps relative order within both subsets + for candidate in sorted(result.mapping.values(), key=lambda c: c.name != c.project_name): ireq = candidate.get_install_requirement() if ireq is None: if candidate.name != candidate.project_name: From 0de374e4df5707955fc6bf9602ee86ea21c8f258 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 6 Sep 2023 13:52:41 +0200 Subject: [PATCH 089/156] review comment: return iterator instead of list --- .../resolution/resolvelib/factory.py | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 0eb7a1c66..81f482c86 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -1,4 +1,5 @@ import contextlib +import itertools import functools import logging from typing import ( @@ -447,7 +448,7 @@ class Factory: def _make_requirements_from_install_req( self, ireq: InstallRequirement, requested_extras: Iterable[str] - ) -> List[Requirement]: + ) -> Iterator[Requirement]: """ Returns requirement objects associated with the given InstallRequirement. In most cases this will be a single object but the following special cases exist: @@ -463,34 +464,32 @@ class Factory: ireq.name, ireq.markers, ) - return [] - if not ireq.link: + yield from () + elif not ireq.link: if ireq.extras and ireq.req is not None and ireq.req.specifier: - return [ - SpecifierRequirement(ireq, drop_extras=True), - SpecifierRequirement(ireq), - ] + yield SpecifierRequirement(ireq, drop_extras=True), + yield SpecifierRequirement(ireq) + else: + self._fail_if_link_is_unsupported_wheel(ireq.link) + cand = self._make_candidate_from_link( + ireq.link, + extras=frozenset(ireq.extras), + template=ireq, + name=canonicalize_name(ireq.name) if ireq.name else None, + version=None, + ) + if cand is None: + # There's no way we can satisfy a URL requirement if the underlying + # candidate fails to build. An unnamed URL must be user-supplied, so + # we fail eagerly. If the URL is named, an unsatisfiable requirement + # can make the resolver do the right thing, either backtrack (and + # maybe find some other requirement that's buildable) or raise a + # ResolutionImpossible eventually. + if not ireq.name: + raise self._build_failures[ireq.link] + yield UnsatisfiableRequirement(canonicalize_name(ireq.name)) else: - return [SpecifierRequirement(ireq)] - self._fail_if_link_is_unsupported_wheel(ireq.link) - cand = self._make_candidate_from_link( - ireq.link, - extras=frozenset(ireq.extras), - template=ireq, - name=canonicalize_name(ireq.name) if ireq.name else None, - version=None, - ) - if cand is None: - # There's no way we can satisfy a URL requirement if the underlying - # candidate fails to build. An unnamed URL must be user-supplied, so - # we fail eagerly. If the URL is named, an unsatisfiable requirement - # can make the resolver do the right thing, either backtrack (and - # maybe find some other requirement that's buildable) or raise a - # ResolutionImpossible eventually. - if not ireq.name: - raise self._build_failures[ireq.link] - return [UnsatisfiableRequirement(canonicalize_name(ireq.name))] - return [self.make_requirement_from_candidate(cand)] + yield self.make_requirement_from_candidate(cand) def collect_root_requirements( self, root_ireqs: List[InstallRequirement] @@ -511,13 +510,14 @@ class Factory: else: collected.constraints[name] = Constraint.from_ireq(ireq) else: - reqs = self._make_requirements_from_install_req( - ireq, - requested_extras=(), + reqs = list( + self._make_requirements_from_install_req( + ireq, + requested_extras=(), + ) ) if not reqs: continue - template = reqs[0] if ireq.user_supplied and template.name not in collected.user_requested: collected.user_requested[template.name] = i @@ -543,7 +543,7 @@ class Factory: specifier: str, comes_from: Optional[InstallRequirement], requested_extras: Iterable[str] = (), - ) -> List[Requirement]: + ) -> Iterator[Requirement]: """ Returns requirement objects associated with the given specifier. In most cases this will be a single object but the following special cases exist: From 5a0167902261b97df768cbcf4757b665cd39229e Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 6 Sep 2023 13:54:28 +0200 Subject: [PATCH 090/156] Update src/pip/_internal/req/constructors.py Co-authored-by: Tzu-ping Chung --- src/pip/_internal/req/constructors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index c03ae718e..a40191954 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -76,7 +76,7 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme post: Optional[str] = match.group(3) assert pre is not None and post is not None extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else "" - return Requirement(pre + extras + post) + return Requirement(f"{pre}{extras}{post}") def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: From 4e73e3e96e99f79d8458517278f67e33796a7fd0 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 6 Sep 2023 14:39:51 +0200 Subject: [PATCH 091/156] review comment: subclass instead of constructor flag --- .../resolution/resolvelib/factory.py | 4 ++-- .../resolution/resolvelib/requirements.py | 24 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 81f482c86..af13a3321 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -1,5 +1,4 @@ import contextlib -import itertools import functools import logging from typing import ( @@ -63,6 +62,7 @@ from .requirements import ( ExplicitRequirement, RequiresPythonRequirement, SpecifierRequirement, + SpecifierWithoutExtrasRequirement, UnsatisfiableRequirement, ) @@ -467,7 +467,7 @@ class Factory: yield from () elif not ireq.link: if ireq.extras and ireq.req is not None and ireq.req.specifier: - yield SpecifierRequirement(ireq, drop_extras=True), + yield SpecifierWithoutExtrasRequirement(ireq), yield SpecifierRequirement(ireq) else: self._fail_if_link_is_unsupported_wheel(ireq.link) diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index becbd6c4b..9c2512823 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -41,18 +41,9 @@ class ExplicitRequirement(Requirement): class SpecifierRequirement(Requirement): - def __init__( - self, - ireq: InstallRequirement, - *, - drop_extras: bool = False, - ) -> None: - """ - :param drop_extras: Ignore any extras that are part of the install requirement, - making this a requirement on the base only. - """ + def __init__(self, ireq: InstallRequirement) -> None: assert ireq.link is None, "This is a link, not a specifier" - self._ireq = ireq if not drop_extras else install_req_drop_extras(ireq) + self._ireq = ireq self._extras = frozenset(self._ireq.extras) def __str__(self) -> str: @@ -102,6 +93,17 @@ class SpecifierRequirement(Requirement): return spec.contains(candidate.version, prereleases=True) +class SpecifierWithoutExtrasRequirement(SpecifierRequirement): + """ + Requirement backed by an install requirement on a base package. Trims extras from its install requirement if there are any. + """ + + def __init__(self, ireq: InstallRequirement) -> None: + assert ireq.link is None, "This is a link, not a specifier" + self._ireq = install_req_drop_extras(ireq) + self._extras = frozenset(self._ireq.extras) + + class RequiresPythonRequirement(Requirement): """A requirement representing Requires-Python metadata.""" From 50cd318cefcdd2a451b0a70a3bf3f31a3ecc6b99 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 6 Sep 2023 15:06:19 +0200 Subject: [PATCH 092/156] review comment: renamed and moved up ExtrasCandidate._ireq --- src/pip/_internal/resolution/resolvelib/candidates.py | 9 +++++---- src/pip/_internal/resolution/resolvelib/factory.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 238834841..d658be372 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -427,10 +427,11 @@ class ExtrasCandidate(Candidate): self, base: BaseCandidate, extras: FrozenSet[str], - ireq: Optional[InstallRequirement] = None, + *, + comes_from: Optional[InstallRequirement] = None, ) -> None: """ - :param ireq: the InstallRequirement that led to this candidate, if it + :param ireq: the InstallRequirement that led to this candidate if it differs from the base's InstallRequirement. This will often be the case in the sense that this candidate's requirement has the extras while the base's does not. Unlike the InstallRequirement backed @@ -439,7 +440,7 @@ class ExtrasCandidate(Candidate): """ self.base = base self.extras = extras - self._ireq = ireq + self._comes_from = comes_from if comes_from is not None else self.base._ireq def __str__(self) -> str: name, rest = str(self.base).split(" ", 1) @@ -514,7 +515,7 @@ class ExtrasCandidate(Candidate): for r in self.base.dist.iter_dependencies(valid_extras): yield from factory.make_requirements_from_spec( str(r), - self._ireq if self._ireq is not None else self.base._ireq, + self._comes_from, valid_extras, ) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index af13a3321..8c5a77991 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -142,13 +142,14 @@ class Factory: self, base: BaseCandidate, extras: FrozenSet[str], - ireq: Optional[InstallRequirement] = None, + *, + comes_from: Optional[InstallRequirement] = None, ) -> ExtrasCandidate: cache_key = (id(base), extras) try: candidate = self._extras_candidate_cache[cache_key] except KeyError: - candidate = ExtrasCandidate(base, extras, ireq=ireq) + candidate = ExtrasCandidate(base, extras, comes_from=comes_from) self._extras_candidate_cache[cache_key] = candidate return candidate @@ -165,7 +166,7 @@ class Factory: self._installed_candidate_cache[dist.canonical_name] = base if not extras: return base - return self._make_extras_candidate(base, extras, ireq=template) + return self._make_extras_candidate(base, extras, comes_from=template) def _make_candidate_from_link( self, @@ -227,7 +228,7 @@ class Factory: if not extras: return base - return self._make_extras_candidate(base, extras, ireq=template) + return self._make_extras_candidate(base, extras, comes_from=template) def _iter_found_candidates( self, From f5602fa0b8a26733cc144b5e1449730fdf620c31 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 6 Sep 2023 15:12:17 +0200 Subject: [PATCH 093/156] added message to invariant assertions --- src/pip/_internal/req/constructors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index a40191954..f0f043b00 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -71,10 +71,10 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme flags=re.ASCII, ) # ireq.req is a valid requirement so the regex should always match - assert match is not None + assert match is not None, f"regex match on requirement {req} failed, this should never happen" pre: Optional[str] = match.group(1) post: Optional[str] = match.group(3) - assert pre is not None and post is not None + assert pre is not None and post is not None, f"regex group selection for requirement {req} failed, this should never happen" extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else "" return Requirement(f"{pre}{extras}{post}") From 449522a8286d28d2c88776dca3cc67b3064982d3 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 6 Sep 2023 15:16:22 +0200 Subject: [PATCH 094/156] minor fixes and linting --- src/pip/_internal/req/constructors.py | 8 ++++++-- src/pip/_internal/resolution/resolvelib/factory.py | 2 +- .../_internal/resolution/resolvelib/requirements.py | 3 ++- src/pip/_internal/resolution/resolvelib/resolver.py | 10 +++++++--- tests/unit/resolution_resolvelib/test_requirement.py | 8 ++++---- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index f0f043b00..b52c9a456 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -71,10 +71,14 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme flags=re.ASCII, ) # ireq.req is a valid requirement so the regex should always match - assert match is not None, f"regex match on requirement {req} failed, this should never happen" + assert ( + match is not None + ), f"regex match on requirement {req} failed, this should never happen" pre: Optional[str] = match.group(1) post: Optional[str] = match.group(3) - assert pre is not None and post is not None, f"regex group selection for requirement {req} failed, this should never happen" + assert ( + pre is not None and post is not None + ), f"regex group selection for requirement {req} failed, this should never happen" extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else "" return Requirement(f"{pre}{extras}{post}") diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 8c5a77991..905449f68 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -468,7 +468,7 @@ class Factory: yield from () elif not ireq.link: if ireq.extras and ireq.req is not None and ireq.req.specifier: - yield SpecifierWithoutExtrasRequirement(ireq), + yield SpecifierWithoutExtrasRequirement(ireq) yield SpecifierRequirement(ireq) else: self._fail_if_link_is_unsupported_wheel(ireq.link) diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index 9c2512823..02cdf65f1 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -95,7 +95,8 @@ class SpecifierRequirement(Requirement): class SpecifierWithoutExtrasRequirement(SpecifierRequirement): """ - Requirement backed by an install requirement on a base package. Trims extras from its install requirement if there are any. + Requirement backed by an install requirement on a base package. + Trims extras from its install requirement if there are any. """ def __init__(self, ireq: InstallRequirement) -> None: diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 2e4941da8..c12beef0b 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -104,9 +104,13 @@ class Resolver(BaseResolver): raise error from e req_set = RequirementSet(check_supported_wheels=check_supported_wheels) - # process candidates with extras last to ensure their base equivalent is already in the req_set if appropriate - # Python's sort is stable so using a binary key function keeps relative order within both subsets - for candidate in sorted(result.mapping.values(), key=lambda c: c.name != c.project_name): + # process candidates with extras last to ensure their base equivalent is + # already in the req_set if appropriate. + # Python's sort is stable so using a binary key function keeps relative order + # within both subsets. + for candidate in sorted( + result.mapping.values(), key=lambda c: c.name != c.project_name + ): ireq = candidate.get_install_requirement() if ireq is None: if candidate.name != candidate.project_name: diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index ce48ab16c..642136a54 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -61,7 +61,7 @@ def test_new_resolver_requirement_has_name( ) -> None: """All requirements should have a name""" for spec, name, _ in test_cases: - reqs = factory.make_requirements_from_spec(spec, comes_from=None) + reqs = list(factory.make_requirements_from_spec(spec, comes_from=None)) assert len(reqs) == 1 assert reqs[0].name == name @@ -71,7 +71,7 @@ def test_new_resolver_correct_number_of_matches( ) -> None: """Requirements should return the correct number of candidates""" for spec, _, match_count in test_cases: - reqs = factory.make_requirements_from_spec(spec, comes_from=None) + reqs = list(factory.make_requirements_from_spec(spec, comes_from=None)) assert len(reqs) == 1 req = reqs[0] matches = factory.find_candidates( @@ -89,7 +89,7 @@ def test_new_resolver_candidates_match_requirement( ) -> None: """Candidates returned from find_candidates should satisfy the requirement""" for spec, _, _ in test_cases: - reqs = factory.make_requirements_from_spec(spec, comes_from=None) + reqs = list(factory.make_requirements_from_spec(spec, comes_from=None)) assert len(reqs) == 1 req = reqs[0] candidates = factory.find_candidates( @@ -106,7 +106,7 @@ def test_new_resolver_candidates_match_requirement( def test_new_resolver_full_resolve(factory: Factory, provider: PipProvider) -> None: """A very basic full resolve""" - reqs = factory.make_requirements_from_spec("simplewheel", comes_from=None) + reqs = list(factory.make_requirements_from_spec("simplewheel", comes_from=None)) assert len(reqs) == 1 r: Resolver[Requirement, Candidate, str] = Resolver(provider, BaseReporter()) result = r.resolve([reqs[0]]) From d5e3f0c4b4d6aa4b432cd5480abb234e2e3332fb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Sep 2023 11:54:00 -0400 Subject: [PATCH 095/156] Use versionchanged syntax --- docs/html/topics/caching.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/html/topics/caching.md b/docs/html/topics/caching.md index 19bd064a7..8d6c40f11 100644 --- a/docs/html/topics/caching.md +++ b/docs/html/topics/caching.md @@ -27,11 +27,12 @@ While this cache attempts to minimize network activity, it does not prevent network access altogether. If you want a local install solution that circumvents accessing PyPI, see {ref}`Installing from local packages`. -In versions prior to 23.2, this cache was stored in a directory called `http` in -the main cache directory (see below for its location). In 23.2 and later, a new -cache format is used, stored in a directory called `http-v2`. If you have -completely switched to newer versions of `pip`, you may wish to delete the old -directory. +```{versionchanged} 23.3 +A new cache format is now used, stored in a directory called `http-v2` (see +below for this directory's location). Previously this cache was stored in a +directory called `http` in the main cache directory. If you have completely +switched to newer versions of `pip`, you may wish to delete the old directory. +``` (wheel-caching)= From b273cee6c5b3572390a3fe9316b2e86661934ce9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Sep 2023 16:42:38 -0400 Subject: [PATCH 096/156] Combine one entry, explain difference between entries better. --- src/pip/_internal/commands/cache.py | 16 ++++++++-------- tests/functional/test_cache.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 83efabe87..0b3380da0 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -96,18 +96,19 @@ class CacheCommand(Command): http_cache_location = self._cache_dir(options, "http-v2") old_http_cache_location = self._cache_dir(options, "http") wheels_cache_location = self._cache_dir(options, "wheels") - http_cache_size = filesystem.format_directory_size(http_cache_location) - old_http_cache_size = filesystem.format_directory_size(old_http_cache_location) + http_cache_size = ( + filesystem.format_directory_size(http_cache_location) + + filesystem.format_directory_size(old_http_cache_location) + ) wheels_cache_size = filesystem.format_directory_size(wheels_cache_location) message = ( textwrap.dedent( """ - Package index page cache location (new): {http_cache_location} - Package index page cache location (old): {old_http_cache_location} - Package index page cache size (new): {http_cache_size} - Package index page cache size (old): {old_http_cache_size} - Number of HTTP files (old+new cache): {num_http_files} + Package index page cache location (pip v23.3+): {http_cache_location} + Package index page cache location (older pips): {old_http_cache_location} + Package index page cache size: {http_cache_size} + Number of HTTP files: {num_http_files} Locally built wheels location: {wheels_cache_location} Locally built wheels size: {wheels_cache_size} Number of locally built wheels: {package_count} @@ -117,7 +118,6 @@ class CacheCommand(Command): http_cache_location=http_cache_location, old_http_cache_location=old_http_cache_location, http_cache_size=http_cache_size, - old_http_cache_size=old_http_cache_size, num_http_files=num_http_files, wheels_cache_location=wheels_cache_location, package_count=num_packages, diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index ddafd7332..c5d910d45 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -203,7 +203,7 @@ def test_cache_info( ) -> None: result = script.pip("cache", "info") - assert f"Package index page cache location (new): {http_cache_dir}" in result.stdout + assert f"Package index page cache location (pip v23.3+): {http_cache_dir}" in result.stdout assert f"Locally built wheels location: {wheel_cache_dir}" in result.stdout num_wheels = len(wheel_cache_files) assert f"Number of locally built wheels: {num_wheels}" in result.stdout From 2951666df5042dc6a329e017e9befcf2b54c25d4 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Thu, 7 Sep 2023 01:17:57 +0200 Subject: [PATCH 097/156] Exclude PR #9634 reformatting from Git blame --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c7644d0e6..f09b08660 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -33,3 +33,4 @@ c7ee560e00b85f7486b452c14ff49e4737996eda # Blacken tools/ 1897784d59e0d5fcda2dd75fea54ddd8be3d502a # Blacken src/pip/_internal/index 94999255d5ede440c37137d210666fdf64302e75 # Reformat the codebase, with black 585037a80a1177f1fa92e159a7079855782e543e # Cleanup implicit string concatenation +8a6f6ac19b80a6dc35900a47016c851d9fcd2ee2 # Blacken src/pip/_internal/resolution directory From 952ab6d837fbece1a221a1d0409eb27f2bb8c544 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Thu, 7 Sep 2023 10:31:49 +0200 Subject: [PATCH 098/156] Update src/pip/_internal/resolution/resolvelib/factory.py Co-authored-by: Tzu-ping Chung --- src/pip/_internal/resolution/resolvelib/factory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 905449f68..2b51aab67 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -465,7 +465,6 @@ class Factory: ireq.name, ireq.markers, ) - yield from () elif not ireq.link: if ireq.extras and ireq.req is not None and ireq.req.specifier: yield SpecifierWithoutExtrasRequirement(ireq) From ab9f6f37f125401f547cd5df84c66d1fb50e4203 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 7 Sep 2023 12:07:50 -0400 Subject: [PATCH 099/156] Fix formatting, combine numbers not strings! Co-authored-by: Pradyun Gedam --- src/pip/_internal/commands/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 0b3380da0..32d1a221d 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -97,8 +97,8 @@ class CacheCommand(Command): old_http_cache_location = self._cache_dir(options, "http") wheels_cache_location = self._cache_dir(options, "wheels") http_cache_size = ( - filesystem.format_directory_size(http_cache_location) + - filesystem.format_directory_size(old_http_cache_location) + filesystem.format_size(filesystem.directory_size(http_cache_location) + + filesystem.directory_size(old_http_cache_location)) ) wheels_cache_size = filesystem.format_directory_size(wheels_cache_location) From fbda0a2ba7e6676f286e515c11b77ded8de996b0 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Fri, 8 Sep 2023 16:32:42 +0200 Subject: [PATCH 100/156] Update tests/unit/resolution_resolvelib/test_requirement.py Co-authored-by: Pradyun Gedam --- tests/unit/resolution_resolvelib/test_requirement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index 642136a54..b8cd13cb5 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -109,5 +109,5 @@ def test_new_resolver_full_resolve(factory: Factory, provider: PipProvider) -> N reqs = list(factory.make_requirements_from_spec("simplewheel", comes_from=None)) assert len(reqs) == 1 r: Resolver[Requirement, Candidate, str] = Resolver(provider, BaseReporter()) - result = r.resolve([reqs[0]]) + result = r.resolve(reqs) assert set(result.mapping.keys()) == {"simplewheel"} From 83ca10ab6012bab3654728b335d3d3b56ac6da06 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Sun, 10 Sep 2023 11:25:34 +0300 Subject: [PATCH 101/156] Update search command docs (#12271) --- docs/html/cli/pip_search.rst | 6 ++++++ news/12059.doc.rst | 1 + 2 files changed, 7 insertions(+) create mode 100644 news/12059.doc.rst diff --git a/docs/html/cli/pip_search.rst b/docs/html/cli/pip_search.rst index 9905a1baf..93ddab3fa 100644 --- a/docs/html/cli/pip_search.rst +++ b/docs/html/cli/pip_search.rst @@ -21,6 +21,12 @@ Usage Description =========== +.. attention:: + PyPI no longer supports ``pip search`` (or XML-RPC search). Please use https://pypi.org/search (via a browser) + instead. See https://warehouse.pypa.io/api-reference/xml-rpc.html#deprecated-methods for more information. + + However, XML-RPC search (and this command) may still be supported by indexes other than PyPI. + .. pip-command-description:: search diff --git a/news/12059.doc.rst b/news/12059.doc.rst new file mode 100644 index 000000000..bf3a8d3e6 --- /dev/null +++ b/news/12059.doc.rst @@ -0,0 +1 @@ +Document that ``pip search`` support has been removed from PyPI From dc188a87e43d7ce1debfe4ed3557ed4023d32504 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 12 Sep 2023 14:11:48 +0800 Subject: [PATCH 102/156] Skip test failing on new Python/setuptools combo This is a temporary measure until we fix the importlib.metadata backend. --- tests/functional/test_install_extras.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index c6cef00fa..db4a811e0 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -150,6 +150,10 @@ def test_install_fails_if_extra_at_end( assert "Extras after version" in result.stderr +@pytest.mark.skipif( + "sys.version_info >= (3, 11)", + reason="Setuptools incompatibility with importlib.metadata; see GH-12267", +) def test_install_special_extra(script: PipTestEnvironment) -> None: # Check that uppercase letters and '-' are dealt with # make a dummy project From c94d81a36de643d2a1176430452a06862a77f58d Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 12 Sep 2023 16:00:40 +0800 Subject: [PATCH 103/156] Setuptools now implements proper normalization --- tests/functional/test_install_extras.py | 12 +----------- tests/requirements-common_wheels.txt | 3 ++- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index 21da9d50e..209429397 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -159,17 +159,7 @@ def test_install_fails_if_extra_at_end( "specified_extra, requested_extra", [ ("Hop_hOp-hoP", "Hop_hOp-hoP"), - pytest.param( - "Hop_hOp-hoP", - "hop-hop-hop", - marks=pytest.mark.xfail( - reason=( - "matching a normalized extra request against an" - "unnormalized extra in metadata requires PEP 685 support " - "in packaging (see pypa/pip#11445)." - ), - ), - ), + ("Hop_hOp-hoP", "hop-hop-hop"), ("hop-hop-hop", "Hop_hOp-hoP"), ], ) diff --git a/tests/requirements-common_wheels.txt b/tests/requirements-common_wheels.txt index 6403ed738..939a111a0 100644 --- a/tests/requirements-common_wheels.txt +++ b/tests/requirements-common_wheels.txt @@ -5,7 +5,8 @@ # 4. Replacing the `setuptools` entry below with a `file:///...` URL # (Adjust artifact directory used based on preference and operating system) -setuptools >= 40.8.0, != 60.6.0 +# Implements new extra normalization. +setuptools >= 68.2 wheel # As required by pytest-cov. coverage >= 4.4 From 9ee4b8ce36ce3f0f57615db017334a43a97d2dea Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 26 Jun 2023 22:18:08 -0500 Subject: [PATCH 104/156] Vendor truststore --- docs/html/topics/https-certificates.md | 12 +- news/truststore.vendor.rst | 1 + src/pip/_internal/cli/req_command.py | 2 +- src/pip/_vendor/__init__.py | 1 + src/pip/_vendor/truststore/LICENSE | 21 + src/pip/_vendor/truststore/__init__.py | 13 + src/pip/_vendor/truststore/_api.py | 302 ++++++++++ src/pip/_vendor/truststore/_macos.py | 501 +++++++++++++++++ src/pip/_vendor/truststore/_openssl.py | 66 +++ src/pip/_vendor/truststore/_ssl_constants.py | 12 + src/pip/_vendor/truststore/_windows.py | 554 +++++++++++++++++++ src/pip/_vendor/truststore/py.typed | 0 src/pip/_vendor/vendor.txt | 1 + tests/functional/test_truststore.py | 15 - 14 files changed, 1474 insertions(+), 27 deletions(-) create mode 100644 news/truststore.vendor.rst create mode 100644 src/pip/_vendor/truststore/LICENSE create mode 100644 src/pip/_vendor/truststore/__init__.py create mode 100644 src/pip/_vendor/truststore/_api.py create mode 100644 src/pip/_vendor/truststore/_macos.py create mode 100644 src/pip/_vendor/truststore/_openssl.py create mode 100644 src/pip/_vendor/truststore/_ssl_constants.py create mode 100644 src/pip/_vendor/truststore/_windows.py create mode 100644 src/pip/_vendor/truststore/py.typed diff --git a/docs/html/topics/https-certificates.md b/docs/html/topics/https-certificates.md index b42c463e6..341cfc632 100644 --- a/docs/html/topics/https-certificates.md +++ b/docs/html/topics/https-certificates.md @@ -28,19 +28,9 @@ It is possible to use the system trust store, instead of the bundled certifi certificates for verifying HTTPS certificates. This approach will typically support corporate proxy certificates without additional configuration. -In order to use system trust stores, you need to: - -- Use Python 3.10 or newer. -- Install the {pypi}`truststore` package, in the Python environment you're - running pip in. - - This is typically done by installing this package using a system package - manager or by using pip in {ref}`Hash-checking mode` for this package and - trusting the network using the `--trusted-host` flag. +In order to use system trust stores, you need to use Python 3.10 or newer. ```{pip-cli} - $ python -m pip install truststore - [...] $ python -m pip install SomePackage --use-feature=truststore [...] Successfully installed SomePackage diff --git a/news/truststore.vendor.rst b/news/truststore.vendor.rst new file mode 100644 index 000000000..ee974728d --- /dev/null +++ b/news/truststore.vendor.rst @@ -0,0 +1 @@ +Add truststore 0.7.0 diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 86070f10c..80b35a80a 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -58,7 +58,7 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]: return None try: - import truststore + from ..._vendor import truststore except ImportError: raise CommandError( "To use the truststore feature, 'truststore' must be installed into " diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index b22f7abb9..c1884baf3 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -117,4 +117,5 @@ if DEBUNDLED: vendored("rich.traceback") vendored("tenacity") vendored("tomli") + vendored("truststore") vendored("urllib3") diff --git a/src/pip/_vendor/truststore/LICENSE b/src/pip/_vendor/truststore/LICENSE new file mode 100644 index 000000000..7ec568c11 --- /dev/null +++ b/src/pip/_vendor/truststore/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2022 Seth Michael Larson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/src/pip/_vendor/truststore/__init__.py b/src/pip/_vendor/truststore/__init__.py new file mode 100644 index 000000000..0f3a4d9e1 --- /dev/null +++ b/src/pip/_vendor/truststore/__init__.py @@ -0,0 +1,13 @@ +"""Verify certificates using native system trust stores""" + +import sys as _sys + +if _sys.version_info < (3, 10): + raise ImportError("truststore requires Python 3.10 or later") + +from ._api import SSLContext, extract_from_ssl, inject_into_ssl # noqa: E402 + +del _api, _sys # type: ignore[name-defined] # noqa: F821 + +__all__ = ["SSLContext", "inject_into_ssl", "extract_from_ssl"] +__version__ = "0.7.0" diff --git a/src/pip/_vendor/truststore/_api.py b/src/pip/_vendor/truststore/_api.py new file mode 100644 index 000000000..264704241 --- /dev/null +++ b/src/pip/_vendor/truststore/_api.py @@ -0,0 +1,302 @@ +import array +import ctypes +import mmap +import os +import pickle +import platform +import socket +import ssl +import typing + +import _ssl # type: ignore[import] + +from ._ssl_constants import _original_SSLContext, _original_super_SSLContext + +if platform.system() == "Windows": + from ._windows import _configure_context, _verify_peercerts_impl +elif platform.system() == "Darwin": + from ._macos import _configure_context, _verify_peercerts_impl +else: + from ._openssl import _configure_context, _verify_peercerts_impl + +# From typeshed/stdlib/ssl.pyi +_StrOrBytesPath: typing.TypeAlias = str | bytes | os.PathLike[str] | os.PathLike[bytes] +_PasswordType: typing.TypeAlias = str | bytes | typing.Callable[[], str | bytes] + +# From typeshed/stdlib/_typeshed/__init__.py +_ReadableBuffer: typing.TypeAlias = typing.Union[ + bytes, + memoryview, + bytearray, + "array.array[typing.Any]", + mmap.mmap, + "ctypes._CData", + pickle.PickleBuffer, +] + + +def inject_into_ssl() -> None: + """Injects the :class:`truststore.SSLContext` into the ``ssl`` + module by replacing :class:`ssl.SSLContext`. + """ + setattr(ssl, "SSLContext", SSLContext) + # urllib3 holds on to its own reference of ssl.SSLContext + # so we need to replace that reference too. + try: + import pip._vendor.urllib3.util.ssl_ as urllib3_ssl + + setattr(urllib3_ssl, "SSLContext", SSLContext) + except ImportError: + pass + + +def extract_from_ssl() -> None: + """Restores the :class:`ssl.SSLContext` class to its original state""" + setattr(ssl, "SSLContext", _original_SSLContext) + try: + import pip._vendor.urllib3.util.ssl_ as urllib3_ssl + + urllib3_ssl.SSLContext = _original_SSLContext + except ImportError: + pass + + +class SSLContext(ssl.SSLContext): + """SSLContext API that uses system certificates on all platforms""" + + def __init__(self, protocol: int = None) -> None: # type: ignore[assignment] + self._ctx = _original_SSLContext(protocol) + + class TruststoreSSLObject(ssl.SSLObject): + # This object exists because wrap_bio() doesn't + # immediately do the handshake so we need to do + # certificate verifications after SSLObject.do_handshake() + + def do_handshake(self) -> None: + ret = super().do_handshake() + _verify_peercerts(self, server_hostname=self.server_hostname) + return ret + + self._ctx.sslobject_class = TruststoreSSLObject + + def wrap_socket( + self, + sock: socket.socket, + server_side: bool = False, + do_handshake_on_connect: bool = True, + suppress_ragged_eofs: bool = True, + server_hostname: str | None = None, + session: ssl.SSLSession | None = None, + ) -> ssl.SSLSocket: + # Use a context manager here because the + # inner SSLContext holds on to our state + # but also does the actual handshake. + with _configure_context(self._ctx): + ssl_sock = self._ctx.wrap_socket( + sock, + server_side=server_side, + server_hostname=server_hostname, + do_handshake_on_connect=do_handshake_on_connect, + suppress_ragged_eofs=suppress_ragged_eofs, + session=session, + ) + try: + _verify_peercerts(ssl_sock, server_hostname=server_hostname) + except Exception: + ssl_sock.close() + raise + return ssl_sock + + def wrap_bio( + self, + incoming: ssl.MemoryBIO, + outgoing: ssl.MemoryBIO, + server_side: bool = False, + server_hostname: str | None = None, + session: ssl.SSLSession | None = None, + ) -> ssl.SSLObject: + with _configure_context(self._ctx): + ssl_obj = self._ctx.wrap_bio( + incoming, + outgoing, + server_hostname=server_hostname, + server_side=server_side, + session=session, + ) + return ssl_obj + + def load_verify_locations( + self, + cafile: str | bytes | os.PathLike[str] | os.PathLike[bytes] | None = None, + capath: str | bytes | os.PathLike[str] | os.PathLike[bytes] | None = None, + cadata: str | _ReadableBuffer | None = None, + ) -> None: + return self._ctx.load_verify_locations( + cafile=cafile, capath=capath, cadata=cadata + ) + + def load_cert_chain( + self, + certfile: _StrOrBytesPath, + keyfile: _StrOrBytesPath | None = None, + password: _PasswordType | None = None, + ) -> None: + return self._ctx.load_cert_chain( + certfile=certfile, keyfile=keyfile, password=password + ) + + def load_default_certs( + self, purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH + ) -> None: + return self._ctx.load_default_certs(purpose) + + def set_alpn_protocols(self, alpn_protocols: typing.Iterable[str]) -> None: + return self._ctx.set_alpn_protocols(alpn_protocols) + + def set_npn_protocols(self, npn_protocols: typing.Iterable[str]) -> None: + return self._ctx.set_npn_protocols(npn_protocols) + + def set_ciphers(self, __cipherlist: str) -> None: + return self._ctx.set_ciphers(__cipherlist) + + def get_ciphers(self) -> typing.Any: + return self._ctx.get_ciphers() + + def session_stats(self) -> dict[str, int]: + return self._ctx.session_stats() + + def cert_store_stats(self) -> dict[str, int]: + raise NotImplementedError() + + @typing.overload + def get_ca_certs( + self, binary_form: typing.Literal[False] = ... + ) -> list[typing.Any]: + ... + + @typing.overload + def get_ca_certs(self, binary_form: typing.Literal[True] = ...) -> list[bytes]: + ... + + @typing.overload + def get_ca_certs(self, binary_form: bool = ...) -> typing.Any: + ... + + def get_ca_certs(self, binary_form: bool = False) -> list[typing.Any] | list[bytes]: + raise NotImplementedError() + + @property + def check_hostname(self) -> bool: + return self._ctx.check_hostname + + @check_hostname.setter + def check_hostname(self, value: bool) -> None: + self._ctx.check_hostname = value + + @property + def hostname_checks_common_name(self) -> bool: + return self._ctx.hostname_checks_common_name + + @hostname_checks_common_name.setter + def hostname_checks_common_name(self, value: bool) -> None: + self._ctx.hostname_checks_common_name = value + + @property + def keylog_filename(self) -> str: + return self._ctx.keylog_filename + + @keylog_filename.setter + def keylog_filename(self, value: str) -> None: + self._ctx.keylog_filename = value + + @property + def maximum_version(self) -> ssl.TLSVersion: + return self._ctx.maximum_version + + @maximum_version.setter + def maximum_version(self, value: ssl.TLSVersion) -> None: + _original_super_SSLContext.maximum_version.__set__( # type: ignore[attr-defined] + self._ctx, value + ) + + @property + def minimum_version(self) -> ssl.TLSVersion: + return self._ctx.minimum_version + + @minimum_version.setter + def minimum_version(self, value: ssl.TLSVersion) -> None: + _original_super_SSLContext.minimum_version.__set__( # type: ignore[attr-defined] + self._ctx, value + ) + + @property + def options(self) -> ssl.Options: + return self._ctx.options + + @options.setter + def options(self, value: ssl.Options) -> None: + _original_super_SSLContext.options.__set__( # type: ignore[attr-defined] + self._ctx, value + ) + + @property + def post_handshake_auth(self) -> bool: + return self._ctx.post_handshake_auth + + @post_handshake_auth.setter + def post_handshake_auth(self, value: bool) -> None: + self._ctx.post_handshake_auth = value + + @property + def protocol(self) -> ssl._SSLMethod: + return self._ctx.protocol + + @property + def security_level(self) -> int: # type: ignore[override] + return self._ctx.security_level + + @property + def verify_flags(self) -> ssl.VerifyFlags: + return self._ctx.verify_flags + + @verify_flags.setter + def verify_flags(self, value: ssl.VerifyFlags) -> None: + _original_super_SSLContext.verify_flags.__set__( # type: ignore[attr-defined] + self._ctx, value + ) + + @property + def verify_mode(self) -> ssl.VerifyMode: + return self._ctx.verify_mode + + @verify_mode.setter + def verify_mode(self, value: ssl.VerifyMode) -> None: + _original_super_SSLContext.verify_mode.__set__( # type: ignore[attr-defined] + self._ctx, value + ) + + +def _verify_peercerts( + sock_or_sslobj: ssl.SSLSocket | ssl.SSLObject, server_hostname: str | None +) -> None: + """ + Verifies the peer certificates from an SSLSocket or SSLObject + against the certificates in the OS trust store. + """ + sslobj: ssl.SSLObject = sock_or_sslobj # type: ignore[assignment] + try: + while not hasattr(sslobj, "get_unverified_chain"): + sslobj = sslobj._sslobj # type: ignore[attr-defined] + except AttributeError: + pass + + # SSLObject.get_unverified_chain() returns 'None' + # if the peer sends no certificates. This is common + # for the server-side scenario. + unverified_chain: typing.Sequence[_ssl.Certificate] = ( + sslobj.get_unverified_chain() or () # type: ignore[attr-defined] + ) + cert_bytes = [cert.public_bytes(_ssl.ENCODING_DER) for cert in unverified_chain] + _verify_peercerts_impl( + sock_or_sslobj.context, cert_bytes, server_hostname=server_hostname + ) diff --git a/src/pip/_vendor/truststore/_macos.py b/src/pip/_vendor/truststore/_macos.py new file mode 100644 index 000000000..7dc440bf3 --- /dev/null +++ b/src/pip/_vendor/truststore/_macos.py @@ -0,0 +1,501 @@ +import contextlib +import ctypes +import platform +import ssl +import typing +from ctypes import ( + CDLL, + POINTER, + c_bool, + c_char_p, + c_int32, + c_long, + c_uint32, + c_ulong, + c_void_p, +) +from ctypes.util import find_library + +from ._ssl_constants import _set_ssl_context_verify_mode + +_mac_version = platform.mac_ver()[0] +_mac_version_info = tuple(map(int, _mac_version.split("."))) +if _mac_version_info < (10, 8): + raise ImportError( + f"Only OS X 10.8 and newer are supported, not {_mac_version_info[0]}.{_mac_version_info[1]}" + ) + + +def _load_cdll(name: str, macos10_16_path: str) -> CDLL: + """Loads a CDLL by name, falling back to known path on 10.16+""" + try: + # Big Sur is technically 11 but we use 10.16 due to the Big Sur + # beta being labeled as 10.16. + path: str | None + if _mac_version_info >= (10, 16): + path = macos10_16_path + else: + path = find_library(name) + if not path: + raise OSError # Caught and reraised as 'ImportError' + return CDLL(path, use_errno=True) + except OSError: + raise ImportError(f"The library {name} failed to load") from None + + +Security = _load_cdll( + "Security", "/System/Library/Frameworks/Security.framework/Security" +) +CoreFoundation = _load_cdll( + "CoreFoundation", + "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", +) + +Boolean = c_bool +CFIndex = c_long +CFStringEncoding = c_uint32 +CFData = c_void_p +CFString = c_void_p +CFArray = c_void_p +CFMutableArray = c_void_p +CFError = c_void_p +CFType = c_void_p +CFTypeID = c_ulong +CFTypeRef = POINTER(CFType) +CFAllocatorRef = c_void_p + +OSStatus = c_int32 + +CFErrorRef = POINTER(CFError) +CFDataRef = POINTER(CFData) +CFStringRef = POINTER(CFString) +CFArrayRef = POINTER(CFArray) +CFMutableArrayRef = POINTER(CFMutableArray) +CFArrayCallBacks = c_void_p +CFOptionFlags = c_uint32 + +SecCertificateRef = POINTER(c_void_p) +SecPolicyRef = POINTER(c_void_p) +SecTrustRef = POINTER(c_void_p) +SecTrustResultType = c_uint32 +SecTrustOptionFlags = c_uint32 + +try: + Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef] + Security.SecCertificateCreateWithData.restype = SecCertificateRef + + Security.SecCertificateCopyData.argtypes = [SecCertificateRef] + Security.SecCertificateCopyData.restype = CFDataRef + + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] + Security.SecCopyErrorMessageString.restype = CFStringRef + + Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef] + Security.SecTrustSetAnchorCertificates.restype = OSStatus + + Security.SecTrustSetAnchorCertificatesOnly.argtypes = [SecTrustRef, Boolean] + Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus + + Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)] + Security.SecTrustEvaluate.restype = OSStatus + + Security.SecPolicyCreateRevocation.argtypes = [CFOptionFlags] + Security.SecPolicyCreateRevocation.restype = SecPolicyRef + + Security.SecPolicyCreateSSL.argtypes = [Boolean, CFStringRef] + Security.SecPolicyCreateSSL.restype = SecPolicyRef + + Security.SecTrustCreateWithCertificates.argtypes = [ + CFTypeRef, + CFTypeRef, + POINTER(SecTrustRef), + ] + Security.SecTrustCreateWithCertificates.restype = OSStatus + + Security.SecTrustGetTrustResult.argtypes = [ + SecTrustRef, + POINTER(SecTrustResultType), + ] + Security.SecTrustGetTrustResult.restype = OSStatus + + Security.SecTrustRef = SecTrustRef # type: ignore[attr-defined] + Security.SecTrustResultType = SecTrustResultType # type: ignore[attr-defined] + Security.OSStatus = OSStatus # type: ignore[attr-defined] + + kSecRevocationUseAnyAvailableMethod = 3 + kSecRevocationRequirePositiveResponse = 8 + + CoreFoundation.CFRelease.argtypes = [CFTypeRef] + CoreFoundation.CFRelease.restype = None + + CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef] + CoreFoundation.CFGetTypeID.restype = CFTypeID + + CoreFoundation.CFStringCreateWithCString.argtypes = [ + CFAllocatorRef, + c_char_p, + CFStringEncoding, + ] + CoreFoundation.CFStringCreateWithCString.restype = CFStringRef + + CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding] + CoreFoundation.CFStringGetCStringPtr.restype = c_char_p + + CoreFoundation.CFStringGetCString.argtypes = [ + CFStringRef, + c_char_p, + CFIndex, + CFStringEncoding, + ] + CoreFoundation.CFStringGetCString.restype = c_bool + + CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex] + CoreFoundation.CFDataCreate.restype = CFDataRef + + CoreFoundation.CFDataGetLength.argtypes = [CFDataRef] + CoreFoundation.CFDataGetLength.restype = CFIndex + + CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef] + CoreFoundation.CFDataGetBytePtr.restype = c_void_p + + CoreFoundation.CFArrayCreate.argtypes = [ + CFAllocatorRef, + POINTER(CFTypeRef), + CFIndex, + CFArrayCallBacks, + ] + CoreFoundation.CFArrayCreate.restype = CFArrayRef + + CoreFoundation.CFArrayCreateMutable.argtypes = [ + CFAllocatorRef, + CFIndex, + CFArrayCallBacks, + ] + CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef + + CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p] + CoreFoundation.CFArrayAppendValue.restype = None + + CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef] + CoreFoundation.CFArrayGetCount.restype = CFIndex + + CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex] + CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p + + CoreFoundation.CFErrorGetCode.argtypes = [CFErrorRef] + CoreFoundation.CFErrorGetCode.restype = CFIndex + + CoreFoundation.CFErrorCopyDescription.argtypes = [CFErrorRef] + CoreFoundation.CFErrorCopyDescription.restype = CFStringRef + + CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( # type: ignore[attr-defined] + CoreFoundation, "kCFAllocatorDefault" + ) + CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( # type: ignore[attr-defined] + CoreFoundation, "kCFTypeArrayCallBacks" + ) + + CoreFoundation.CFTypeRef = CFTypeRef # type: ignore[attr-defined] + CoreFoundation.CFArrayRef = CFArrayRef # type: ignore[attr-defined] + CoreFoundation.CFStringRef = CFStringRef # type: ignore[attr-defined] + CoreFoundation.CFErrorRef = CFErrorRef # type: ignore[attr-defined] + +except AttributeError: + raise ImportError("Error initializing ctypes") from None + + +def _handle_osstatus(result: OSStatus, _: typing.Any, args: typing.Any) -> typing.Any: + """ + Raises an error if the OSStatus value is non-zero. + """ + if int(result) == 0: + return args + + # Returns a CFString which we need to transform + # into a UTF-8 Python string. + error_message_cfstring = None + try: + error_message_cfstring = Security.SecCopyErrorMessageString(result, None) + + # First step is convert the CFString into a C string pointer. + # We try the fast no-copy way first. + error_message_cfstring_c_void_p = ctypes.cast( + error_message_cfstring, ctypes.POINTER(ctypes.c_void_p) + ) + message = CoreFoundation.CFStringGetCStringPtr( + error_message_cfstring_c_void_p, CFConst.kCFStringEncodingUTF8 + ) + + # Quoting the Apple dev docs: + # + # "A pointer to a C string or NULL if the internal + # storage of theString does not allow this to be + # returned efficiently." + # + # So we need to get our hands dirty. + if message is None: + buffer = ctypes.create_string_buffer(1024) + result = CoreFoundation.CFStringGetCString( + error_message_cfstring_c_void_p, + buffer, + 1024, + CFConst.kCFStringEncodingUTF8, + ) + if not result: + raise OSError("Error copying C string from CFStringRef") + message = buffer.value + + finally: + if error_message_cfstring is not None: + CoreFoundation.CFRelease(error_message_cfstring) + + # If no message can be found for this status we come + # up with a generic one that forwards the status code. + if message is None or message == "": + message = f"SecureTransport operation returned a non-zero OSStatus: {result}" + + raise ssl.SSLError(message) + + +Security.SecTrustCreateWithCertificates.errcheck = _handle_osstatus # type: ignore[assignment] +Security.SecTrustSetAnchorCertificates.errcheck = _handle_osstatus # type: ignore[assignment] +Security.SecTrustGetTrustResult.errcheck = _handle_osstatus # type: ignore[assignment] + + +class CFConst: + """CoreFoundation constants""" + + kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) + + errSecIncompleteCertRevocationCheck = -67635 + errSecHostNameMismatch = -67602 + errSecCertificateExpired = -67818 + errSecNotTrusted = -67843 + + +def _bytes_to_cf_data_ref(value: bytes) -> CFDataRef: # type: ignore[valid-type] + return CoreFoundation.CFDataCreate( # type: ignore[no-any-return] + CoreFoundation.kCFAllocatorDefault, value, len(value) + ) + + +def _bytes_to_cf_string(value: bytes) -> CFString: + """ + Given a Python binary data, create a CFString. + The string must be CFReleased by the caller. + """ + c_str = ctypes.c_char_p(value) + cf_str = CoreFoundation.CFStringCreateWithCString( + CoreFoundation.kCFAllocatorDefault, + c_str, + CFConst.kCFStringEncodingUTF8, + ) + return cf_str # type: ignore[no-any-return] + + +def _cf_string_ref_to_str(cf_string_ref: CFStringRef) -> str | None: # type: ignore[valid-type] + """ + Creates a Unicode string from a CFString object. Used entirely for error + reporting. + Yes, it annoys me quite a lot that this function is this complex. + """ + + string = CoreFoundation.CFStringGetCStringPtr( + cf_string_ref, CFConst.kCFStringEncodingUTF8 + ) + if string is None: + buffer = ctypes.create_string_buffer(1024) + result = CoreFoundation.CFStringGetCString( + cf_string_ref, buffer, 1024, CFConst.kCFStringEncodingUTF8 + ) + if not result: + raise OSError("Error copying C string from CFStringRef") + string = buffer.value + if string is not None: + string = string.decode("utf-8") + return string # type: ignore[no-any-return] + + +def _der_certs_to_cf_cert_array(certs: list[bytes]) -> CFMutableArrayRef: # type: ignore[valid-type] + """Builds a CFArray of SecCertificateRefs from a list of DER-encoded certificates. + Responsibility of the caller to call CoreFoundation.CFRelease on the CFArray. + """ + cf_array = CoreFoundation.CFArrayCreateMutable( + CoreFoundation.kCFAllocatorDefault, + 0, + ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), + ) + if not cf_array: + raise MemoryError("Unable to allocate memory!") + + for cert_data in certs: + cf_data = None + sec_cert_ref = None + try: + cf_data = _bytes_to_cf_data_ref(cert_data) + sec_cert_ref = Security.SecCertificateCreateWithData( + CoreFoundation.kCFAllocatorDefault, cf_data + ) + CoreFoundation.CFArrayAppendValue(cf_array, sec_cert_ref) + finally: + if cf_data: + CoreFoundation.CFRelease(cf_data) + if sec_cert_ref: + CoreFoundation.CFRelease(sec_cert_ref) + + return cf_array # type: ignore[no-any-return] + + +@contextlib.contextmanager +def _configure_context(ctx: ssl.SSLContext) -> typing.Iterator[None]: + check_hostname = ctx.check_hostname + verify_mode = ctx.verify_mode + ctx.check_hostname = False + _set_ssl_context_verify_mode(ctx, ssl.CERT_NONE) + try: + yield + finally: + ctx.check_hostname = check_hostname + _set_ssl_context_verify_mode(ctx, verify_mode) + + +def _verify_peercerts_impl( + ssl_context: ssl.SSLContext, + cert_chain: list[bytes], + server_hostname: str | None = None, +) -> None: + certs = None + policies = None + trust = None + cf_error = None + try: + if server_hostname is not None: + cf_str_hostname = None + try: + cf_str_hostname = _bytes_to_cf_string(server_hostname.encode("ascii")) + ssl_policy = Security.SecPolicyCreateSSL(True, cf_str_hostname) + finally: + if cf_str_hostname: + CoreFoundation.CFRelease(cf_str_hostname) + else: + ssl_policy = Security.SecPolicyCreateSSL(True, None) + + policies = ssl_policy + if ssl_context.verify_flags & ssl.VERIFY_CRL_CHECK_CHAIN: + # Add explicit policy requiring positive revocation checks + policies = CoreFoundation.CFArrayCreateMutable( + CoreFoundation.kCFAllocatorDefault, + 0, + ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), + ) + CoreFoundation.CFArrayAppendValue(policies, ssl_policy) + CoreFoundation.CFRelease(ssl_policy) + revocation_policy = Security.SecPolicyCreateRevocation( + kSecRevocationUseAnyAvailableMethod + | kSecRevocationRequirePositiveResponse + ) + CoreFoundation.CFArrayAppendValue(policies, revocation_policy) + CoreFoundation.CFRelease(revocation_policy) + elif ssl_context.verify_flags & ssl.VERIFY_CRL_CHECK_LEAF: + raise NotImplementedError("VERIFY_CRL_CHECK_LEAF not implemented for macOS") + + certs = None + try: + certs = _der_certs_to_cf_cert_array(cert_chain) + + # Now that we have certificates loaded and a SecPolicy + # we can finally create a SecTrust object! + trust = Security.SecTrustRef() + Security.SecTrustCreateWithCertificates( + certs, policies, ctypes.byref(trust) + ) + + finally: + # The certs are now being held by SecTrust so we can + # release our handles for the array. + if certs: + CoreFoundation.CFRelease(certs) + + # If there are additional trust anchors to load we need to transform + # the list of DER-encoded certificates into a CFArray. Otherwise + # pass 'None' to signal that we only want system / fetched certificates. + ctx_ca_certs_der: list[bytes] | None = ssl_context.get_ca_certs( + binary_form=True + ) + if ctx_ca_certs_der: + ctx_ca_certs = None + try: + ctx_ca_certs = _der_certs_to_cf_cert_array(cert_chain) + Security.SecTrustSetAnchorCertificates(trust, ctx_ca_certs) + finally: + if ctx_ca_certs: + CoreFoundation.CFRelease(ctx_ca_certs) + else: + Security.SecTrustSetAnchorCertificates(trust, None) + + cf_error = CoreFoundation.CFErrorRef() + sec_trust_eval_result = Security.SecTrustEvaluateWithError( + trust, ctypes.byref(cf_error) + ) + # sec_trust_eval_result is a bool (0 or 1) + # where 1 means that the certs are trusted. + if sec_trust_eval_result == 1: + is_trusted = True + elif sec_trust_eval_result == 0: + is_trusted = False + else: + raise ssl.SSLError( + f"Unknown result from Security.SecTrustEvaluateWithError: {sec_trust_eval_result!r}" + ) + + cf_error_code = 0 + if not is_trusted: + cf_error_code = CoreFoundation.CFErrorGetCode(cf_error) + + # If the error is a known failure that we're + # explicitly okay with from SSLContext configuration + # we can set is_trusted accordingly. + if ssl_context.verify_mode != ssl.CERT_REQUIRED and ( + cf_error_code == CFConst.errSecNotTrusted + or cf_error_code == CFConst.errSecCertificateExpired + ): + is_trusted = True + elif ( + not ssl_context.check_hostname + and cf_error_code == CFConst.errSecHostNameMismatch + ): + is_trusted = True + + # If we're still not trusted then we start to + # construct and raise the SSLCertVerificationError. + if not is_trusted: + cf_error_string_ref = None + try: + cf_error_string_ref = CoreFoundation.CFErrorCopyDescription(cf_error) + + # Can this ever return 'None' if there's a CFError? + cf_error_message = ( + _cf_string_ref_to_str(cf_error_string_ref) + or "Certificate verification failed" + ) + + # TODO: Not sure if we need the SecTrustResultType for anything? + # We only care whether or not it's a success or failure for now. + sec_trust_result_type = Security.SecTrustResultType() + Security.SecTrustGetTrustResult( + trust, ctypes.byref(sec_trust_result_type) + ) + + err = ssl.SSLCertVerificationError(cf_error_message) + err.verify_message = cf_error_message + err.verify_code = cf_error_code + raise err + finally: + if cf_error_string_ref: + CoreFoundation.CFRelease(cf_error_string_ref) + + finally: + if policies: + CoreFoundation.CFRelease(policies) + if trust: + CoreFoundation.CFRelease(trust) diff --git a/src/pip/_vendor/truststore/_openssl.py b/src/pip/_vendor/truststore/_openssl.py new file mode 100644 index 000000000..9951cf75c --- /dev/null +++ b/src/pip/_vendor/truststore/_openssl.py @@ -0,0 +1,66 @@ +import contextlib +import os +import re +import ssl +import typing + +# candidates based on https://github.com/tiran/certifi-system-store by Christian Heimes +_CA_FILE_CANDIDATES = [ + # Alpine, Arch, Fedora 34+, OpenWRT, RHEL 9+, BSD + "/etc/ssl/cert.pem", + # Fedora <= 34, RHEL <= 9, CentOS <= 9 + "/etc/pki/tls/cert.pem", + # Debian, Ubuntu (requires ca-certificates) + "/etc/ssl/certs/ca-certificates.crt", + # SUSE + "/etc/ssl/ca-bundle.pem", +] + +_HASHED_CERT_FILENAME_RE = re.compile(r"^[0-9a-fA-F]{8}\.[0-9]$") + + +@contextlib.contextmanager +def _configure_context(ctx: ssl.SSLContext) -> typing.Iterator[None]: + # First, check whether the default locations from OpenSSL + # seem like they will give us a usable set of CA certs. + # ssl.get_default_verify_paths already takes care of: + # - getting cafile from either the SSL_CERT_FILE env var + # or the path configured when OpenSSL was compiled, + # and verifying that that path exists + # - getting capath from either the SSL_CERT_DIR env var + # or the path configured when OpenSSL was compiled, + # and verifying that that path exists + # In addition we'll check whether capath appears to contain certs. + defaults = ssl.get_default_verify_paths() + if defaults.cafile or (defaults.capath and _capath_contains_certs(defaults.capath)): + ctx.set_default_verify_paths() + else: + # cafile from OpenSSL doesn't exist + # and capath from OpenSSL doesn't contain certs. + # Let's search other common locations instead. + for cafile in _CA_FILE_CANDIDATES: + if os.path.isfile(cafile): + ctx.load_verify_locations(cafile=cafile) + break + + yield + + +def _capath_contains_certs(capath: str) -> bool: + """Check whether capath exists and contains certs in the expected format.""" + if not os.path.isdir(capath): + return False + for name in os.listdir(capath): + if _HASHED_CERT_FILENAME_RE.match(name): + return True + return False + + +def _verify_peercerts_impl( + ssl_context: ssl.SSLContext, + cert_chain: list[bytes], + server_hostname: str | None = None, +) -> None: + # This is a no-op because we've enabled SSLContext's built-in + # verification via verify_mode=CERT_REQUIRED, and don't need to repeat it. + pass diff --git a/src/pip/_vendor/truststore/_ssl_constants.py b/src/pip/_vendor/truststore/_ssl_constants.py new file mode 100644 index 000000000..be60f8301 --- /dev/null +++ b/src/pip/_vendor/truststore/_ssl_constants.py @@ -0,0 +1,12 @@ +import ssl + +# Hold on to the original class so we can create it consistently +# even if we inject our own SSLContext into the ssl module. +_original_SSLContext = ssl.SSLContext +_original_super_SSLContext = super(_original_SSLContext, _original_SSLContext) + + +def _set_ssl_context_verify_mode( + ssl_context: ssl.SSLContext, verify_mode: ssl.VerifyMode +) -> None: + _original_super_SSLContext.verify_mode.__set__(ssl_context, verify_mode) # type: ignore[attr-defined] diff --git a/src/pip/_vendor/truststore/_windows.py b/src/pip/_vendor/truststore/_windows.py new file mode 100644 index 000000000..3de4960a1 --- /dev/null +++ b/src/pip/_vendor/truststore/_windows.py @@ -0,0 +1,554 @@ +import contextlib +import ssl +import typing +from ctypes import WinDLL # type: ignore +from ctypes import WinError # type: ignore +from ctypes import ( + POINTER, + Structure, + c_char_p, + c_ulong, + c_void_p, + c_wchar_p, + cast, + create_unicode_buffer, + pointer, + sizeof, +) +from ctypes.wintypes import ( + BOOL, + DWORD, + HANDLE, + LONG, + LPCSTR, + LPCVOID, + LPCWSTR, + LPFILETIME, + LPSTR, + LPWSTR, +) +from typing import TYPE_CHECKING, Any + +from ._ssl_constants import _set_ssl_context_verify_mode + +HCERTCHAINENGINE = HANDLE +HCERTSTORE = HANDLE +HCRYPTPROV_LEGACY = HANDLE + + +class CERT_CONTEXT(Structure): + _fields_ = ( + ("dwCertEncodingType", DWORD), + ("pbCertEncoded", c_void_p), + ("cbCertEncoded", DWORD), + ("pCertInfo", c_void_p), + ("hCertStore", HCERTSTORE), + ) + + +PCERT_CONTEXT = POINTER(CERT_CONTEXT) +PCCERT_CONTEXT = POINTER(PCERT_CONTEXT) + + +class CERT_ENHKEY_USAGE(Structure): + _fields_ = ( + ("cUsageIdentifier", DWORD), + ("rgpszUsageIdentifier", POINTER(LPSTR)), + ) + + +PCERT_ENHKEY_USAGE = POINTER(CERT_ENHKEY_USAGE) + + +class CERT_USAGE_MATCH(Structure): + _fields_ = ( + ("dwType", DWORD), + ("Usage", CERT_ENHKEY_USAGE), + ) + + +class CERT_CHAIN_PARA(Structure): + _fields_ = ( + ("cbSize", DWORD), + ("RequestedUsage", CERT_USAGE_MATCH), + ("RequestedIssuancePolicy", CERT_USAGE_MATCH), + ("dwUrlRetrievalTimeout", DWORD), + ("fCheckRevocationFreshnessTime", BOOL), + ("dwRevocationFreshnessTime", DWORD), + ("pftCacheResync", LPFILETIME), + ("pStrongSignPara", c_void_p), + ("dwStrongSignFlags", DWORD), + ) + + +if TYPE_CHECKING: + PCERT_CHAIN_PARA = pointer[CERT_CHAIN_PARA] # type: ignore[misc] +else: + PCERT_CHAIN_PARA = POINTER(CERT_CHAIN_PARA) + + +class CERT_TRUST_STATUS(Structure): + _fields_ = ( + ("dwErrorStatus", DWORD), + ("dwInfoStatus", DWORD), + ) + + +class CERT_CHAIN_ELEMENT(Structure): + _fields_ = ( + ("cbSize", DWORD), + ("pCertContext", PCERT_CONTEXT), + ("TrustStatus", CERT_TRUST_STATUS), + ("pRevocationInfo", c_void_p), + ("pIssuanceUsage", PCERT_ENHKEY_USAGE), + ("pApplicationUsage", PCERT_ENHKEY_USAGE), + ("pwszExtendedErrorInfo", LPCWSTR), + ) + + +PCERT_CHAIN_ELEMENT = POINTER(CERT_CHAIN_ELEMENT) + + +class CERT_SIMPLE_CHAIN(Structure): + _fields_ = ( + ("cbSize", DWORD), + ("TrustStatus", CERT_TRUST_STATUS), + ("cElement", DWORD), + ("rgpElement", POINTER(PCERT_CHAIN_ELEMENT)), + ("pTrustListInfo", c_void_p), + ("fHasRevocationFreshnessTime", BOOL), + ("dwRevocationFreshnessTime", DWORD), + ) + + +PCERT_SIMPLE_CHAIN = POINTER(CERT_SIMPLE_CHAIN) + + +class CERT_CHAIN_CONTEXT(Structure): + _fields_ = ( + ("cbSize", DWORD), + ("TrustStatus", CERT_TRUST_STATUS), + ("cChain", DWORD), + ("rgpChain", POINTER(PCERT_SIMPLE_CHAIN)), + ("cLowerQualityChainContext", DWORD), + ("rgpLowerQualityChainContext", c_void_p), + ("fHasRevocationFreshnessTime", BOOL), + ("dwRevocationFreshnessTime", DWORD), + ) + + +PCERT_CHAIN_CONTEXT = POINTER(CERT_CHAIN_CONTEXT) +PCCERT_CHAIN_CONTEXT = POINTER(PCERT_CHAIN_CONTEXT) + + +class SSL_EXTRA_CERT_CHAIN_POLICY_PARA(Structure): + _fields_ = ( + ("cbSize", DWORD), + ("dwAuthType", DWORD), + ("fdwChecks", DWORD), + ("pwszServerName", LPCWSTR), + ) + + +class CERT_CHAIN_POLICY_PARA(Structure): + _fields_ = ( + ("cbSize", DWORD), + ("dwFlags", DWORD), + ("pvExtraPolicyPara", c_void_p), + ) + + +PCERT_CHAIN_POLICY_PARA = POINTER(CERT_CHAIN_POLICY_PARA) + + +class CERT_CHAIN_POLICY_STATUS(Structure): + _fields_ = ( + ("cbSize", DWORD), + ("dwError", DWORD), + ("lChainIndex", LONG), + ("lElementIndex", LONG), + ("pvExtraPolicyStatus", c_void_p), + ) + + +PCERT_CHAIN_POLICY_STATUS = POINTER(CERT_CHAIN_POLICY_STATUS) + + +class CERT_CHAIN_ENGINE_CONFIG(Structure): + _fields_ = ( + ("cbSize", DWORD), + ("hRestrictedRoot", HCERTSTORE), + ("hRestrictedTrust", HCERTSTORE), + ("hRestrictedOther", HCERTSTORE), + ("cAdditionalStore", DWORD), + ("rghAdditionalStore", c_void_p), + ("dwFlags", DWORD), + ("dwUrlRetrievalTimeout", DWORD), + ("MaximumCachedCertificates", DWORD), + ("CycleDetectionModulus", DWORD), + ("hExclusiveRoot", HCERTSTORE), + ("hExclusiveTrustedPeople", HCERTSTORE), + ("dwExclusiveFlags", DWORD), + ) + + +PCERT_CHAIN_ENGINE_CONFIG = POINTER(CERT_CHAIN_ENGINE_CONFIG) +PHCERTCHAINENGINE = POINTER(HCERTCHAINENGINE) + +X509_ASN_ENCODING = 0x00000001 +PKCS_7_ASN_ENCODING = 0x00010000 +CERT_STORE_PROV_MEMORY = b"Memory" +CERT_STORE_ADD_USE_EXISTING = 2 +USAGE_MATCH_TYPE_OR = 1 +OID_PKIX_KP_SERVER_AUTH = c_char_p(b"1.3.6.1.5.5.7.3.1") +CERT_CHAIN_REVOCATION_CHECK_END_CERT = 0x10000000 +CERT_CHAIN_REVOCATION_CHECK_CHAIN = 0x20000000 +CERT_CHAIN_POLICY_IGNORE_ALL_NOT_TIME_VALID_FLAGS = 0x00000007 +CERT_CHAIN_POLICY_IGNORE_INVALID_BASIC_CONSTRAINTS_FLAG = 0x00000008 +CERT_CHAIN_POLICY_ALLOW_UNKNOWN_CA_FLAG = 0x00000010 +CERT_CHAIN_POLICY_IGNORE_INVALID_NAME_FLAG = 0x00000040 +CERT_CHAIN_POLICY_IGNORE_WRONG_USAGE_FLAG = 0x00000020 +CERT_CHAIN_POLICY_IGNORE_INVALID_POLICY_FLAG = 0x00000080 +CERT_CHAIN_POLICY_IGNORE_ALL_REV_UNKNOWN_FLAGS = 0x00000F00 +CERT_CHAIN_POLICY_ALLOW_TESTROOT_FLAG = 0x00008000 +CERT_CHAIN_POLICY_TRUST_TESTROOT_FLAG = 0x00004000 +AUTHTYPE_SERVER = 2 +CERT_CHAIN_POLICY_SSL = 4 +FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000 +FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200 + +# Flags to set for SSLContext.verify_mode=CERT_NONE +CERT_CHAIN_POLICY_VERIFY_MODE_NONE_FLAGS = ( + CERT_CHAIN_POLICY_IGNORE_ALL_NOT_TIME_VALID_FLAGS + | CERT_CHAIN_POLICY_IGNORE_INVALID_BASIC_CONSTRAINTS_FLAG + | CERT_CHAIN_POLICY_ALLOW_UNKNOWN_CA_FLAG + | CERT_CHAIN_POLICY_IGNORE_INVALID_NAME_FLAG + | CERT_CHAIN_POLICY_IGNORE_WRONG_USAGE_FLAG + | CERT_CHAIN_POLICY_IGNORE_INVALID_POLICY_FLAG + | CERT_CHAIN_POLICY_IGNORE_ALL_REV_UNKNOWN_FLAGS + | CERT_CHAIN_POLICY_ALLOW_TESTROOT_FLAG + | CERT_CHAIN_POLICY_TRUST_TESTROOT_FLAG +) + +wincrypt = WinDLL("crypt32.dll") +kernel32 = WinDLL("kernel32.dll") + + +def _handle_win_error(result: bool, _: Any, args: Any) -> Any: + if not result: + # Note, actually raises OSError after calling GetLastError and FormatMessage + raise WinError() + return args + + +CertCreateCertificateChainEngine = wincrypt.CertCreateCertificateChainEngine +CertCreateCertificateChainEngine.argtypes = ( + PCERT_CHAIN_ENGINE_CONFIG, + PHCERTCHAINENGINE, +) +CertCreateCertificateChainEngine.errcheck = _handle_win_error + +CertOpenStore = wincrypt.CertOpenStore +CertOpenStore.argtypes = (LPCSTR, DWORD, HCRYPTPROV_LEGACY, DWORD, c_void_p) +CertOpenStore.restype = HCERTSTORE +CertOpenStore.errcheck = _handle_win_error + +CertAddEncodedCertificateToStore = wincrypt.CertAddEncodedCertificateToStore +CertAddEncodedCertificateToStore.argtypes = ( + HCERTSTORE, + DWORD, + c_char_p, + DWORD, + DWORD, + PCCERT_CONTEXT, +) +CertAddEncodedCertificateToStore.restype = BOOL + +CertCreateCertificateContext = wincrypt.CertCreateCertificateContext +CertCreateCertificateContext.argtypes = (DWORD, c_char_p, DWORD) +CertCreateCertificateContext.restype = PCERT_CONTEXT +CertCreateCertificateContext.errcheck = _handle_win_error + +CertGetCertificateChain = wincrypt.CertGetCertificateChain +CertGetCertificateChain.argtypes = ( + HCERTCHAINENGINE, + PCERT_CONTEXT, + LPFILETIME, + HCERTSTORE, + PCERT_CHAIN_PARA, + DWORD, + c_void_p, + PCCERT_CHAIN_CONTEXT, +) +CertGetCertificateChain.restype = BOOL +CertGetCertificateChain.errcheck = _handle_win_error + +CertVerifyCertificateChainPolicy = wincrypt.CertVerifyCertificateChainPolicy +CertVerifyCertificateChainPolicy.argtypes = ( + c_ulong, + PCERT_CHAIN_CONTEXT, + PCERT_CHAIN_POLICY_PARA, + PCERT_CHAIN_POLICY_STATUS, +) +CertVerifyCertificateChainPolicy.restype = BOOL + +CertCloseStore = wincrypt.CertCloseStore +CertCloseStore.argtypes = (HCERTSTORE, DWORD) +CertCloseStore.restype = BOOL +CertCloseStore.errcheck = _handle_win_error + +CertFreeCertificateChain = wincrypt.CertFreeCertificateChain +CertFreeCertificateChain.argtypes = (PCERT_CHAIN_CONTEXT,) + +CertFreeCertificateContext = wincrypt.CertFreeCertificateContext +CertFreeCertificateContext.argtypes = (PCERT_CONTEXT,) + +CertFreeCertificateChainEngine = wincrypt.CertFreeCertificateChainEngine +CertFreeCertificateChainEngine.argtypes = (HCERTCHAINENGINE,) + +FormatMessageW = kernel32.FormatMessageW +FormatMessageW.argtypes = ( + DWORD, + LPCVOID, + DWORD, + DWORD, + LPWSTR, + DWORD, + c_void_p, +) +FormatMessageW.restype = DWORD + + +def _verify_peercerts_impl( + ssl_context: ssl.SSLContext, + cert_chain: list[bytes], + server_hostname: str | None = None, +) -> None: + """Verify the cert_chain from the server using Windows APIs.""" + pCertContext = None + hIntermediateCertStore = CertOpenStore(CERT_STORE_PROV_MEMORY, 0, None, 0, None) + try: + # Add intermediate certs to an in-memory cert store + for cert_bytes in cert_chain[1:]: + CertAddEncodedCertificateToStore( + hIntermediateCertStore, + X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, + cert_bytes, + len(cert_bytes), + CERT_STORE_ADD_USE_EXISTING, + None, + ) + + # Cert context for leaf cert + leaf_cert = cert_chain[0] + pCertContext = CertCreateCertificateContext( + X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, leaf_cert, len(leaf_cert) + ) + + # Chain params to match certs for serverAuth extended usage + cert_enhkey_usage = CERT_ENHKEY_USAGE() + cert_enhkey_usage.cUsageIdentifier = 1 + cert_enhkey_usage.rgpszUsageIdentifier = (c_char_p * 1)(OID_PKIX_KP_SERVER_AUTH) + cert_usage_match = CERT_USAGE_MATCH() + cert_usage_match.Usage = cert_enhkey_usage + chain_params = CERT_CHAIN_PARA() + chain_params.RequestedUsage = cert_usage_match + chain_params.cbSize = sizeof(chain_params) + pChainPara = pointer(chain_params) + + if ssl_context.verify_flags & ssl.VERIFY_CRL_CHECK_CHAIN: + chain_flags = CERT_CHAIN_REVOCATION_CHECK_CHAIN + elif ssl_context.verify_flags & ssl.VERIFY_CRL_CHECK_LEAF: + chain_flags = CERT_CHAIN_REVOCATION_CHECK_END_CERT + else: + chain_flags = 0 + + try: + # First attempt to verify using the default Windows system trust roots + # (default chain engine). + _get_and_verify_cert_chain( + ssl_context, + None, + hIntermediateCertStore, + pCertContext, + pChainPara, + server_hostname, + chain_flags=chain_flags, + ) + except ssl.SSLCertVerificationError: + # If that fails but custom CA certs have been added + # to the SSLContext using load_verify_locations, + # try verifying using a custom chain engine + # that trusts the custom CA certs. + custom_ca_certs: list[bytes] | None = ssl_context.get_ca_certs( + binary_form=True + ) + if custom_ca_certs: + _verify_using_custom_ca_certs( + ssl_context, + custom_ca_certs, + hIntermediateCertStore, + pCertContext, + pChainPara, + server_hostname, + chain_flags=chain_flags, + ) + else: + raise + finally: + CertCloseStore(hIntermediateCertStore, 0) + if pCertContext: + CertFreeCertificateContext(pCertContext) + + +def _get_and_verify_cert_chain( + ssl_context: ssl.SSLContext, + hChainEngine: HCERTCHAINENGINE | None, + hIntermediateCertStore: HCERTSTORE, + pPeerCertContext: c_void_p, + pChainPara: PCERT_CHAIN_PARA, # type: ignore[valid-type] + server_hostname: str | None, + chain_flags: int, +) -> None: + ppChainContext = None + try: + # Get cert chain + ppChainContext = pointer(PCERT_CHAIN_CONTEXT()) + CertGetCertificateChain( + hChainEngine, # chain engine + pPeerCertContext, # leaf cert context + None, # current system time + hIntermediateCertStore, # additional in-memory cert store + pChainPara, # chain-building parameters + chain_flags, + None, # reserved + ppChainContext, # the resulting chain context + ) + pChainContext = ppChainContext.contents + + # Verify cert chain + ssl_extra_cert_chain_policy_para = SSL_EXTRA_CERT_CHAIN_POLICY_PARA() + ssl_extra_cert_chain_policy_para.cbSize = sizeof( + ssl_extra_cert_chain_policy_para + ) + ssl_extra_cert_chain_policy_para.dwAuthType = AUTHTYPE_SERVER + ssl_extra_cert_chain_policy_para.fdwChecks = 0 + if server_hostname: + ssl_extra_cert_chain_policy_para.pwszServerName = c_wchar_p(server_hostname) + + chain_policy = CERT_CHAIN_POLICY_PARA() + chain_policy.pvExtraPolicyPara = cast( + pointer(ssl_extra_cert_chain_policy_para), c_void_p + ) + if ssl_context.verify_mode == ssl.CERT_NONE: + chain_policy.dwFlags |= CERT_CHAIN_POLICY_VERIFY_MODE_NONE_FLAGS + if not ssl_context.check_hostname: + chain_policy.dwFlags |= CERT_CHAIN_POLICY_IGNORE_INVALID_NAME_FLAG + chain_policy.cbSize = sizeof(chain_policy) + + pPolicyPara = pointer(chain_policy) + policy_status = CERT_CHAIN_POLICY_STATUS() + policy_status.cbSize = sizeof(policy_status) + pPolicyStatus = pointer(policy_status) + CertVerifyCertificateChainPolicy( + CERT_CHAIN_POLICY_SSL, + pChainContext, + pPolicyPara, + pPolicyStatus, + ) + + # Check status + error_code = policy_status.dwError + if error_code: + # Try getting a human readable message for an error code. + error_message_buf = create_unicode_buffer(1024) + error_message_chars = FormatMessageW( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + None, + error_code, + 0, + error_message_buf, + sizeof(error_message_buf), + None, + ) + + # See if we received a message for the error, + # otherwise we use a generic error with the + # error code and hope that it's search-able. + if error_message_chars <= 0: + error_message = f"Certificate chain policy error {error_code:#x} [{policy_status.lElementIndex}]" + else: + error_message = error_message_buf.value.strip() + + err = ssl.SSLCertVerificationError(error_message) + err.verify_message = error_message + err.verify_code = error_code + raise err from None + finally: + if ppChainContext: + CertFreeCertificateChain(ppChainContext.contents) + + +def _verify_using_custom_ca_certs( + ssl_context: ssl.SSLContext, + custom_ca_certs: list[bytes], + hIntermediateCertStore: HCERTSTORE, + pPeerCertContext: c_void_p, + pChainPara: PCERT_CHAIN_PARA, # type: ignore[valid-type] + server_hostname: str | None, + chain_flags: int, +) -> None: + hChainEngine = None + hRootCertStore = CertOpenStore(CERT_STORE_PROV_MEMORY, 0, None, 0, None) + try: + # Add custom CA certs to an in-memory cert store + for cert_bytes in custom_ca_certs: + CertAddEncodedCertificateToStore( + hRootCertStore, + X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, + cert_bytes, + len(cert_bytes), + CERT_STORE_ADD_USE_EXISTING, + None, + ) + + # Create a custom cert chain engine which exclusively trusts + # certs from our hRootCertStore + cert_chain_engine_config = CERT_CHAIN_ENGINE_CONFIG() + cert_chain_engine_config.cbSize = sizeof(cert_chain_engine_config) + cert_chain_engine_config.hExclusiveRoot = hRootCertStore + pConfig = pointer(cert_chain_engine_config) + phChainEngine = pointer(HCERTCHAINENGINE()) + CertCreateCertificateChainEngine( + pConfig, + phChainEngine, + ) + hChainEngine = phChainEngine.contents + + # Get and verify a cert chain using the custom chain engine + _get_and_verify_cert_chain( + ssl_context, + hChainEngine, + hIntermediateCertStore, + pPeerCertContext, + pChainPara, + server_hostname, + chain_flags, + ) + finally: + if hChainEngine: + CertFreeCertificateChainEngine(hChainEngine) + CertCloseStore(hRootCertStore, 0) + + +@contextlib.contextmanager +def _configure_context(ctx: ssl.SSLContext) -> typing.Iterator[None]: + check_hostname = ctx.check_hostname + verify_mode = ctx.verify_mode + ctx.check_hostname = False + _set_ssl_context_verify_mode(ctx, ssl.CERT_NONE) + try: + yield + finally: + ctx.check_hostname = check_hostname + _set_ssl_context_verify_mode(ctx, verify_mode) diff --git a/src/pip/_vendor/truststore/py.typed b/src/pip/_vendor/truststore/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 08e1acb01..56cf75517 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -20,4 +20,5 @@ setuptools==68.0.0 six==1.16.0 tenacity==8.2.2 tomli==2.0.1 +truststore==0.7.0 webencodings==0.5.1 diff --git a/tests/functional/test_truststore.py b/tests/functional/test_truststore.py index 33153d0fb..cc90343b5 100644 --- a/tests/functional/test_truststore.py +++ b/tests/functional/test_truststore.py @@ -27,20 +27,6 @@ def test_truststore_error_on_old_python(pip: PipRunner) -> None: assert "The truststore feature is only available for Python 3.10+" in result.stderr -@pytest.mark.skipif(sys.version_info < (3, 10), reason="3.10+ required for truststore") -def test_truststore_error_without_preinstalled(pip: PipRunner) -> None: - result = pip( - "install", - "--no-index", - "does-not-matter", - expect_error=True, - ) - assert ( - "To use the truststore feature, 'truststore' must be installed into " - "pip's current environment." - ) in result.stderr - - @pytest.mark.skipif(sys.version_info < (3, 10), reason="3.10+ required for truststore") @pytest.mark.network @pytest.mark.parametrize( @@ -56,6 +42,5 @@ def test_trustore_can_install( pip: PipRunner, package: str, ) -> None: - script.pip("install", "truststore") result = pip("install", package) assert "Successfully installed" in result.stdout From 9a65b887a44555ba6b1ad19b8b81c834472ce3b8 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 26 Jun 2023 22:49:06 -0500 Subject: [PATCH 105/156] Use absolute instead of relative imports for vendored modules Co-authored-by: Tzu-ping Chung --- src/pip/_internal/cli/req_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 80b35a80a..a2395d68c 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -58,7 +58,7 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]: return None try: - from ..._vendor import truststore + from pip._vendor import truststore except ImportError: raise CommandError( "To use the truststore feature, 'truststore' must be installed into " From 44857c6e82c3219b39d55c61aded270a480b4f5d Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 26 Jun 2023 22:52:57 -0500 Subject: [PATCH 106/156] Update error message to forward platform-specific error --- src/pip/_internal/cli/req_command.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index a2395d68c..080739aa9 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -59,10 +59,9 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]: try: from pip._vendor import truststore - except ImportError: + except ImportError as e: raise CommandError( - "To use the truststore feature, 'truststore' must be installed into " - "pip's current environment." + f"The truststore feature is unavailable: {e}" ) return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) From 63f19b5eade3891d98fbc419e2a0b686e11752b4 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Sat, 19 Aug 2023 11:49:32 -0500 Subject: [PATCH 107/156] Explicitly require Python 3.10+ for vendoring task --- noxfile.py | 6 ++++++ src/pip/_internal/cli/req_command.py | 4 +--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 041d90399..a3e7ceab4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -184,6 +184,12 @@ def lint(session: nox.Session) -> None: # git reset --hard origin/main @nox.session def vendoring(session: nox.Session) -> None: + # Ensure that the session Python is running 3.10+ + # so that truststore can be installed correctly. + session.run( + "python", "-c", "import sys; sys.exit(1 if sys.version_info < (3, 10) else 0)" + ) + session.install("vendoring~=1.2.0") parser = argparse.ArgumentParser(prog="nox -s vendoring") diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 080739aa9..7a53d5105 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -60,9 +60,7 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]: try: from pip._vendor import truststore except ImportError as e: - raise CommandError( - f"The truststore feature is unavailable: {e}" - ) + raise CommandError(f"The truststore feature is unavailable: {e}") return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) From fca773ccde9a95df9ff8c9153e3497fc13571912 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 22 Aug 2023 20:43:54 -0500 Subject: [PATCH 108/156] Allow truststore to not import on Python 3.9 and earlier --- src/pip/_internal/commands/debug.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index 1b1fd3ea5..ab8280db4 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -46,22 +46,29 @@ def create_vendor_txt_map() -> Dict[str, str]: return dict(line.split("==", 1) for line in lines) -def get_module_from_module_name(module_name: str) -> ModuleType: +def get_module_from_module_name(module_name: str) -> Optional[ModuleType]: # Module name can be uppercase in vendor.txt for some reason... module_name = module_name.lower().replace("-", "_") # PATCH: setuptools is actually only pkg_resources. if module_name == "setuptools": module_name = "pkg_resources" - __import__(f"pip._vendor.{module_name}", globals(), locals(), level=0) - return getattr(pip._vendor, module_name) + try: + __import__(f"pip._vendor.{module_name}", globals(), locals(), level=0) + return getattr(pip._vendor, module_name) + except ImportError: + # We allow 'truststore' to fail to import due + # to being unavailable on Python 3.9 and earlier. + if module_name == "truststore" and sys.version_info < (3, 10): + return None + raise def get_vendor_version_from_module(module_name: str) -> Optional[str]: module = get_module_from_module_name(module_name) version = getattr(module, "__version__", None) - if not version: + if module and not version: # Try to find version in debundled module info. assert module.__file__ is not None env = get_environment([os.path.dirname(module.__file__)]) From bff1e6a67be0545dd7e2ac1cb5189463370c3b55 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 12 Sep 2023 15:55:51 -0500 Subject: [PATCH 109/156] Vendor truststore 0.8.0 --- news/truststore.vendor.rst | 2 +- src/pip/_vendor/truststore/__init__.py | 2 +- src/pip/_vendor/truststore/_api.py | 38 ++++++++++---------- src/pip/_vendor/truststore/_ssl_constants.py | 19 ++++++++++ src/pip/_vendor/vendor.txt | 2 +- 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/news/truststore.vendor.rst b/news/truststore.vendor.rst index ee974728d..63c71d72d 100644 --- a/news/truststore.vendor.rst +++ b/news/truststore.vendor.rst @@ -1 +1 @@ -Add truststore 0.7.0 +Add truststore 0.8.0 diff --git a/src/pip/_vendor/truststore/__init__.py b/src/pip/_vendor/truststore/__init__.py index 0f3a4d9e1..59930f455 100644 --- a/src/pip/_vendor/truststore/__init__.py +++ b/src/pip/_vendor/truststore/__init__.py @@ -10,4 +10,4 @@ from ._api import SSLContext, extract_from_ssl, inject_into_ssl # noqa: E402 del _api, _sys # type: ignore[name-defined] # noqa: F821 __all__ = ["SSLContext", "inject_into_ssl", "extract_from_ssl"] -__version__ = "0.7.0" +__version__ = "0.8.0" diff --git a/src/pip/_vendor/truststore/_api.py b/src/pip/_vendor/truststore/_api.py index 264704241..829aff726 100644 --- a/src/pip/_vendor/truststore/_api.py +++ b/src/pip/_vendor/truststore/_api.py @@ -1,8 +1,4 @@ -import array -import ctypes -import mmap import os -import pickle import platform import socket import ssl @@ -10,7 +6,12 @@ import typing import _ssl # type: ignore[import] -from ._ssl_constants import _original_SSLContext, _original_super_SSLContext +from ._ssl_constants import ( + _original_SSLContext, + _original_super_SSLContext, + _truststore_SSLContext_dunder_class, + _truststore_SSLContext_super_class, +) if platform.system() == "Windows": from ._windows import _configure_context, _verify_peercerts_impl @@ -19,21 +20,13 @@ elif platform.system() == "Darwin": else: from ._openssl import _configure_context, _verify_peercerts_impl +if typing.TYPE_CHECKING: + from pip._vendor.typing_extensions import Buffer + # From typeshed/stdlib/ssl.pyi _StrOrBytesPath: typing.TypeAlias = str | bytes | os.PathLike[str] | os.PathLike[bytes] _PasswordType: typing.TypeAlias = str | bytes | typing.Callable[[], str | bytes] -# From typeshed/stdlib/_typeshed/__init__.py -_ReadableBuffer: typing.TypeAlias = typing.Union[ - bytes, - memoryview, - bytearray, - "array.array[typing.Any]", - mmap.mmap, - "ctypes._CData", - pickle.PickleBuffer, -] - def inject_into_ssl() -> None: """Injects the :class:`truststore.SSLContext` into the ``ssl`` @@ -61,9 +54,16 @@ def extract_from_ssl() -> None: pass -class SSLContext(ssl.SSLContext): +class SSLContext(_truststore_SSLContext_super_class): # type: ignore[misc] """SSLContext API that uses system certificates on all platforms""" + @property # type: ignore[misc] + def __class__(self) -> type: + # Dirty hack to get around isinstance() checks + # for ssl.SSLContext instances in aiohttp/trustme + # when using non-CPython implementations. + return _truststore_SSLContext_dunder_class or SSLContext + def __init__(self, protocol: int = None) -> None: # type: ignore[assignment] self._ctx = _original_SSLContext(protocol) @@ -129,7 +129,7 @@ class SSLContext(ssl.SSLContext): self, cafile: str | bytes | os.PathLike[str] | os.PathLike[bytes] | None = None, capath: str | bytes | os.PathLike[str] | os.PathLike[bytes] | None = None, - cadata: str | _ReadableBuffer | None = None, + cadata: typing.Union[str, "Buffer", None] = None, ) -> None: return self._ctx.load_verify_locations( cafile=cafile, capath=capath, cadata=cadata @@ -252,7 +252,7 @@ class SSLContext(ssl.SSLContext): return self._ctx.protocol @property - def security_level(self) -> int: # type: ignore[override] + def security_level(self) -> int: return self._ctx.security_level @property diff --git a/src/pip/_vendor/truststore/_ssl_constants.py b/src/pip/_vendor/truststore/_ssl_constants.py index be60f8301..b1ee7a3cb 100644 --- a/src/pip/_vendor/truststore/_ssl_constants.py +++ b/src/pip/_vendor/truststore/_ssl_constants.py @@ -1,10 +1,29 @@ import ssl +import sys +import typing # Hold on to the original class so we can create it consistently # even if we inject our own SSLContext into the ssl module. _original_SSLContext = ssl.SSLContext _original_super_SSLContext = super(_original_SSLContext, _original_SSLContext) +# CPython is known to be good, but non-CPython implementations +# may implement SSLContext differently so to be safe we don't +# subclass the SSLContext. + +# This is returned by truststore.SSLContext.__class__() +_truststore_SSLContext_dunder_class: typing.Optional[type] + +# This value is the superclass of truststore.SSLContext. +_truststore_SSLContext_super_class: type + +if sys.implementation.name == "cpython": + _truststore_SSLContext_super_class = _original_SSLContext + _truststore_SSLContext_dunder_class = None +else: + _truststore_SSLContext_super_class = object + _truststore_SSLContext_dunder_class = _original_SSLContext + def _set_ssl_context_verify_mode( ssl_context: ssl.SSLContext, verify_mode: ssl.VerifyMode diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 56cf75517..ade8512e2 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -20,5 +20,5 @@ setuptools==68.0.0 six==1.16.0 tenacity==8.2.2 tomli==2.0.1 -truststore==0.7.0 +truststore==0.8.0 webencodings==0.5.1 From 90c4a4230d0dff833e5e087cd85cebde1c134233 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 13 Sep 2023 12:23:59 +0800 Subject: [PATCH 110/156] Manually build package and revert xfail marker --- tests/functional/test_install_extras.py | 26 ++++++++----------------- tests/requirements-common_wheels.txt | 6 +----- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index 813c95bfa..8ccbcf199 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -163,12 +163,10 @@ def test_install_fails_if_extra_at_end( "Hop_hOp-hoP", "hop-hop-hop", marks=pytest.mark.xfail( - "sys.version_info < (3, 8)", reason=( "matching a normalized extra request against an" "unnormalized extra in metadata requires PEP 685 support " - "in either packaging or the build tool. Setuptools " - "implements this in 68.2, which requires 3.8+" + "in packaging (see pypa/pip#11445)." ), ), ), @@ -180,26 +178,18 @@ def test_install_special_extra( specified_extra: str, requested_extra: str, ) -> None: - # Check that uppercase letters and '-' are dealt with - # make a dummy project - pkga_path = script.scratch_path / "pkga" - pkga_path.mkdir() - pkga_path.joinpath("setup.py").write_text( - textwrap.dedent( - f""" - from setuptools import setup - setup(name='pkga', - version='0.1', - extras_require={{'{specified_extra}': ['missing_pkg']}}, - ) - """ - ) + """Check extra normalization is implemented according to specification.""" + pkga_path = create_basic_wheel_for_package( + script, + name="pkga", + version="0.1", + extras={specified_extra: ["missing_pkg"]}, ) result = script.pip( "install", "--no-index", - f"{pkga_path}[{requested_extra}]", + f"pkga[{requested_extra}] @ {pkga_path.as_uri()}", expect_error=True, ) assert ( diff --git a/tests/requirements-common_wheels.txt b/tests/requirements-common_wheels.txt index 8963e3337..6403ed738 100644 --- a/tests/requirements-common_wheels.txt +++ b/tests/requirements-common_wheels.txt @@ -5,11 +5,7 @@ # 4. Replacing the `setuptools` entry below with a `file:///...` URL # (Adjust artifact directory used based on preference and operating system) -# Implements new extra normalization. -setuptools >= 68.2 ; python_version >= '3.8' -setuptools >= 40.8.0, != 60.6.0 ; python_version < '3.8' - +setuptools >= 40.8.0, != 60.6.0 wheel - # As required by pytest-cov. coverage >= 4.4 From 7127fc96f4dfd7ab9b873664b57318c9fc693e3a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 13 Sep 2023 13:27:11 +0800 Subject: [PATCH 111/156] Prevent eager extra normalization This removes extra normalization when metadata is loaded into the data structures, so we can obtain the raw values later in the process during resolution. The change in match_markers is needed because this is relied on by the legacy resolver. Since we removed eager normalization, we need to do that when the extras are used instead to maintain compatibility. --- src/pip/_internal/metadata/importlib/_dists.py | 7 ++----- src/pip/_internal/req/req_install.py | 5 +++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/pip/_internal/metadata/importlib/_dists.py b/src/pip/_internal/metadata/importlib/_dists.py index 65c043c87..c43ef8d01 100644 --- a/src/pip/_internal/metadata/importlib/_dists.py +++ b/src/pip/_internal/metadata/importlib/_dists.py @@ -27,7 +27,6 @@ from pip._internal.metadata.base import ( Wheel, ) from pip._internal.utils.misc import normalize_path -from pip._internal.utils.packaging import safe_extra from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.wheel import parse_wheel, read_wheel_metadata_file @@ -208,12 +207,10 @@ class Distribution(BaseDistribution): return cast(email.message.Message, self._dist.metadata) def iter_provided_extras(self) -> Iterable[str]: - return ( - safe_extra(extra) for extra in self.metadata.get_all("Provides-Extra", []) - ) + return (extra for extra in self.metadata.get_all("Provides-Extra", [])) def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]: - contexts: Sequence[Dict[str, str]] = [{"extra": safe_extra(e)} for e in extras] + contexts: Sequence[Dict[str, str]] = [{"extra": e} for e in extras] for req_string in self.metadata.get_all("Requires-Dist", []): req = Requirement(req_string) if not req.marker: diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 8110114ca..84f337d6e 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -128,7 +128,7 @@ class InstallRequirement: if extras: self.extras = extras elif req: - self.extras = {safe_extra(extra) for extra in req.extras} + self.extras = req.extras else: self.extras = set() if markers is None and req: @@ -272,7 +272,8 @@ class InstallRequirement: extras_requested = ("",) if self.markers is not None: return any( - self.markers.evaluate({"extra": extra}) for extra in extras_requested + self.markers.evaluate({"extra": safe_extra(extra)}) + for extra in extras_requested ) else: return True From 9ba2bb90fb57e4ee5f624ecd39eade207863c21a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 13 Sep 2023 16:35:59 +0800 Subject: [PATCH 112/156] Straighten up extra comps across metadata backends The importlib.metadata and pkg_resources backends unfortunately normalize extras differently, and we don't really want to continue using the latter's logic (being partially lossy while still not compliant to standards), so we add a new abstraction for the purpose. --- src/pip/_internal/metadata/__init__.py | 3 +- src/pip/_internal/metadata/base.py | 32 +++++++++++++------ .../_internal/metadata/importlib/__init__.py | 4 ++- .../_internal/metadata/importlib/_dists.py | 8 ++++- src/pip/_internal/metadata/pkg_resources.py | 10 +++++- src/pip/_internal/req/req_install.py | 6 +++- .../resolution/resolvelib/candidates.py | 23 ++++++++----- 7 files changed, 64 insertions(+), 22 deletions(-) diff --git a/src/pip/_internal/metadata/__init__.py b/src/pip/_internal/metadata/__init__.py index 9f73ca710..aa232b6ca 100644 --- a/src/pip/_internal/metadata/__init__.py +++ b/src/pip/_internal/metadata/__init__.py @@ -9,7 +9,7 @@ from pip._internal.utils.misc import strtobool from .base import BaseDistribution, BaseEnvironment, FilesystemWheel, MemoryWheel, Wheel if TYPE_CHECKING: - from typing import Protocol + from typing import Literal, Protocol else: Protocol = object @@ -50,6 +50,7 @@ def _should_use_importlib_metadata() -> bool: class Backend(Protocol): + NAME: 'Literal["importlib", "pkg_resources"]' Distribution: Type[BaseDistribution] Environment: Type[BaseEnvironment] diff --git a/src/pip/_internal/metadata/base.py b/src/pip/_internal/metadata/base.py index cafb79fb3..924912441 100644 --- a/src/pip/_internal/metadata/base.py +++ b/src/pip/_internal/metadata/base.py @@ -24,7 +24,7 @@ from typing import ( from pip._vendor.packaging.requirements import Requirement from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet -from pip._vendor.packaging.utils import NormalizedName +from pip._vendor.packaging.utils import NormalizedName, canonicalize_name from pip._vendor.packaging.version import LegacyVersion, Version from pip._internal.exceptions import NoneMetadataError @@ -37,7 +37,6 @@ from pip._internal.models.direct_url import ( from pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here. from pip._internal.utils.egg_link import egg_link_path_from_sys_path from pip._internal.utils.misc import is_local, normalize_path -from pip._internal.utils.packaging import safe_extra from pip._internal.utils.urls import url_to_path from ._json import msg_to_json @@ -460,6 +459,19 @@ class BaseDistribution(Protocol): For modern .dist-info distributions, this is the collection of "Provides-Extra:" entries in distribution metadata. + + The return value of this function is not particularly useful other than + display purposes due to backward compatibility issues and the extra + names being poorly normalized prior to PEP 685. If you want to perform + logic operations on extras, use :func:`is_extra_provided` instead. + """ + raise NotImplementedError() + + def is_extra_provided(self, extra: str) -> bool: + """Check whether an extra is provided by this distribution. + + This is needed mostly for compatibility issues with pkg_resources not + following the extra normalization rules defined in PEP 685. """ raise NotImplementedError() @@ -537,10 +549,11 @@ class BaseDistribution(Protocol): """Get extras from the egg-info directory.""" known_extras = {""} for entry in self._iter_requires_txt_entries(): - if entry.extra in known_extras: + extra = canonicalize_name(entry.extra) + if extra in known_extras: continue - known_extras.add(entry.extra) - yield entry.extra + known_extras.add(extra) + yield extra def _iter_egg_info_dependencies(self) -> Iterable[str]: """Get distribution dependencies from the egg-info directory. @@ -556,10 +569,11 @@ class BaseDistribution(Protocol): all currently available PEP 517 backends, although not standardized. """ for entry in self._iter_requires_txt_entries(): - if entry.extra and entry.marker: - marker = f'({entry.marker}) and extra == "{safe_extra(entry.extra)}"' - elif entry.extra: - marker = f'extra == "{safe_extra(entry.extra)}"' + extra = canonicalize_name(entry.extra) + if extra and entry.marker: + marker = f'({entry.marker}) and extra == "{extra}"' + elif extra: + marker = f'extra == "{extra}"' elif entry.marker: marker = entry.marker else: diff --git a/src/pip/_internal/metadata/importlib/__init__.py b/src/pip/_internal/metadata/importlib/__init__.py index 5e7af9fe5..a779138db 100644 --- a/src/pip/_internal/metadata/importlib/__init__.py +++ b/src/pip/_internal/metadata/importlib/__init__.py @@ -1,4 +1,6 @@ from ._dists import Distribution from ._envs import Environment -__all__ = ["Distribution", "Environment"] +__all__ = ["NAME", "Distribution", "Environment"] + +NAME = "importlib" diff --git a/src/pip/_internal/metadata/importlib/_dists.py b/src/pip/_internal/metadata/importlib/_dists.py index c43ef8d01..26370facf 100644 --- a/src/pip/_internal/metadata/importlib/_dists.py +++ b/src/pip/_internal/metadata/importlib/_dists.py @@ -207,7 +207,13 @@ class Distribution(BaseDistribution): return cast(email.message.Message, self._dist.metadata) def iter_provided_extras(self) -> Iterable[str]: - return (extra for extra in self.metadata.get_all("Provides-Extra", [])) + return self.metadata.get_all("Provides-Extra", []) + + def is_extra_provided(self, extra: str) -> bool: + return any( + canonicalize_name(provided_extra) == canonicalize_name(extra) + for provided_extra in self.metadata.get_all("Provides-Extra", []) + ) def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]: contexts: Sequence[Dict[str, str]] = [{"extra": e} for e in extras] diff --git a/src/pip/_internal/metadata/pkg_resources.py b/src/pip/_internal/metadata/pkg_resources.py index f330ef12a..bb11e5bd8 100644 --- a/src/pip/_internal/metadata/pkg_resources.py +++ b/src/pip/_internal/metadata/pkg_resources.py @@ -24,8 +24,12 @@ from .base import ( Wheel, ) +__all__ = ["NAME", "Distribution", "Environment"] + logger = logging.getLogger(__name__) +NAME = "pkg_resources" + class EntryPoint(NamedTuple): name: str @@ -212,12 +216,16 @@ class Distribution(BaseDistribution): def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]: if extras: # pkg_resources raises on invalid extras, so we sanitize. - extras = frozenset(extras).intersection(self._dist.extras) + extras = frozenset(pkg_resources.safe_extra(e) for e in extras) + extras = extras.intersection(self._dist.extras) return self._dist.requires(extras) def iter_provided_extras(self) -> Iterable[str]: return self._dist.extras + def is_extra_provided(self, extra: str) -> bool: + return pkg_resources.safe_extra(extra) in self._dist.extras + class Environment(BaseEnvironment): def __init__(self, ws: pkg_resources.WorkingSet) -> None: diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 84f337d6e..f8957e5d9 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -272,7 +272,11 @@ class InstallRequirement: extras_requested = ("",) if self.markers is not None: return any( - self.markers.evaluate({"extra": safe_extra(extra)}) + self.markers.evaluate({"extra": extra}) + # TODO: Remove these two variants when packaging is upgraded to + # support the marker comparison logic specified in PEP 685. + or self.markers.evaluate({"extra": safe_extra(extra)}) + or self.markers.evaluate({"extra": canonicalize_name(extra)}) for extra in extras_requested ) else: diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 13204b9f1..67737a509 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -435,7 +435,8 @@ class ExtrasCandidate(Candidate): # since PEP 685 has not been implemented for marker-matching, and using # the non-normalized extra for lookup ensures the user can select a # non-normalized extra in a package with its non-normalized form. - # TODO: Remove this when packaging is upgraded to support PEP 685. + # TODO: Remove this attribute when packaging is upgraded to support the + # marker comparison logic specified in PEP 685. self._unnormalized_extras = extras.difference(self.extras) def __str__(self) -> str: @@ -490,18 +491,20 @@ class ExtrasCandidate(Candidate): def _warn_invalid_extras( self, requested: FrozenSet[str], - provided: FrozenSet[str], + valid: FrozenSet[str], ) -> None: """Emit warnings for invalid extras being requested. This emits a warning for each requested extra that is not in the candidate's ``Provides-Extra`` list. """ - invalid_extras_to_warn = requested.difference( - provided, + invalid_extras_to_warn = frozenset( + extra + for extra in requested + if extra not in valid # If an extra is requested in an unnormalized form, skip warning # about the normalized form being missing. - (canonicalize_name(e) for e in self._unnormalized_extras), + and extra in self.extras ) if not invalid_extras_to_warn: return @@ -521,9 +524,13 @@ class ExtrasCandidate(Candidate): cause a warning to be logged here. """ requested_extras = self.extras.union(self._unnormalized_extras) - provided_extras = frozenset(self.base.dist.iter_provided_extras()) - self._warn_invalid_extras(requested_extras, provided_extras) - return requested_extras.intersection(provided_extras) + valid_extras = frozenset( + extra + for extra in requested_extras + if self.base.dist.is_extra_provided(extra) + ) + self._warn_invalid_extras(requested_extras, valid_extras) + return valid_extras def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]: factory = self.base._factory From ce949466c96086a36aefb8ed1106113fca731fa6 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 13 Sep 2023 15:14:07 +0200 Subject: [PATCH 113/156] fixed argument name in docstring --- src/pip/_internal/resolution/resolvelib/candidates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index d658be372..bf89a515d 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -431,7 +431,7 @@ class ExtrasCandidate(Candidate): comes_from: Optional[InstallRequirement] = None, ) -> None: """ - :param ireq: the InstallRequirement that led to this candidate if it + :param comes_from: the InstallRequirement that led to this candidate if it differs from the base's InstallRequirement. This will often be the case in the sense that this candidate's requirement has the extras while the base's does not. Unlike the InstallRequirement backed From 0f543e3c7e05d40e1ecf684cade068fed1c200f9 Mon Sep 17 00:00:00 2001 From: Sander Van Balen Date: Wed, 13 Sep 2023 16:48:16 +0200 Subject: [PATCH 114/156] made assertions more robust --- tests/functional/test_new_resolver.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index 77dede2fc..b5945edf8 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Callable, Dict, List, Tuple import pytest +from tests.conftest import ScriptFactory from tests.lib import ( PipTestEnvironment, create_basic_sdist_for_package, @@ -13,6 +14,7 @@ from tests.lib import ( create_test_package_with_setup, ) from tests.lib.direct_url import get_created_direct_url +from tests.lib.venv import VirtualEnvironment from tests.lib.wheel import make_wheel if TYPE_CHECKING: @@ -2313,7 +2315,11 @@ def test_new_resolver_dont_backtrack_on_extra_if_base_constrained_in_requirement @pytest.mark.parametrize("swap_order", (True, False)) @pytest.mark.parametrize("two_extras", (True, False)) def test_new_resolver_dont_backtrack_on_conflicting_constraints_on_extras( - script: PipTestEnvironment, swap_order: bool, two_extras: bool + tmpdir: pathlib.Path, + virtualenv: VirtualEnvironment, + script_factory: ScriptFactory, + swap_order: bool, + two_extras: bool, ) -> None: """ Verify that conflicting constraints on the same package with different @@ -2323,6 +2329,11 @@ def test_new_resolver_dont_backtrack_on_conflicting_constraints_on_extras( :param swap_order: swap the order the install specifiers appear in :param two_extras: also add an extra for the second specifier """ + script: PipTestEnvironment = script_factory( + tmpdir.joinpath("workspace"), + virtualenv, + {**os.environ, "PIP_RESOLVER_DEBUG": "1"}, + ) create_basic_wheel_for_package(script, "dep", "1.0") create_basic_wheel_for_package( script, "pkg", "1.0", extras={"ext1": ["dep"], "ext2": ["dep"]} @@ -2348,9 +2359,13 @@ def test_new_resolver_dont_backtrack_on_conflicting_constraints_on_extras( assert ( "pkg-2.0" not in result.stdout or "pkg-1.0" not in result.stdout ), "Should only try one of 1.0, 2.0 depending on order" + assert "Reporter.starting()" in result.stdout, ( + "This should never fail unless the debug reporting format has changed," + " in which case the other assertions in this test need to be reviewed." + ) assert ( - "looking at multiple versions" not in result.stdout - ), "Should not have to look at multiple versions to conclude conflict" + "Reporter.rejecting_candidate" not in result.stdout + ), "Should be able to conclude conflict before even selecting a candidate" assert ( "conflict is caused by" in result.stdout ), "Resolver should be trivially able to find conflict cause" From 3b4738cf9aba08b9fe6dc9a2ac667bff2a2585a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 17 Sep 2023 20:02:47 +0200 Subject: [PATCH 115/156] Fix git version parsing issue --- news/12280.bugfix.rst | 1 + src/pip/_internal/vcs/git.py | 2 +- tests/unit/test_vcs.py | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 news/12280.bugfix.rst diff --git a/news/12280.bugfix.rst b/news/12280.bugfix.rst new file mode 100644 index 000000000..77de283d3 --- /dev/null +++ b/news/12280.bugfix.rst @@ -0,0 +1 @@ +Fix crash when the git version number contains something else than digits and dots. diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 8d1d49937..8c242cf89 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -101,7 +101,7 @@ class Git(VersionControl): if not match: logger.warning("Can't parse git version: %s", version) return () - return tuple(int(c) for c in match.groups()) + return (int(match.group(1)), int(match.group(2))) @classmethod def get_current_branch(cls, location: str) -> Optional[str]: diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 3ecc69abf..fb6c3ea31 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -598,6 +598,21 @@ def test_get_git_version() -> None: assert git_version >= (1, 0, 0) +@pytest.mark.parametrize( + ("version", "expected"), + [ + ("git version 2.17", (2, 17)), + ("git version 2.18.1", (2, 18)), + ("git version 2.35.GIT", (2, 35)), # gh:12280 + ("oh my git version 2.37.GIT", ()), # invalid version + ("git version 2.GIT", ()), # invalid version + ], +) +def test_get_git_version_parser(version: str, expected: Tuple[int, int]) -> None: + with mock.patch("pip._internal.vcs.git.Git.run_command", return_value=version): + assert Git().get_git_version() == expected + + @pytest.mark.parametrize( "use_interactive,is_atty,expected", [ From 184e4826269da5caa73c8f4114fb00cd1678c075 Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Wed, 20 Sep 2023 18:48:52 -0400 Subject: [PATCH 116/156] Clarify --prefer-binary --- src/pip/_internal/cli/cmdoptions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 64bc59bbd..84b40a8fc 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -670,7 +670,10 @@ def prefer_binary() -> Option: dest="prefer_binary", action="store_true", default=False, - help="Prefer older binary packages over newer source packages.", + help=( + "Prefer binary packages over source packages, even if the " + "source packages are newer." + ), ) From 677c3eed9fd6c150d9ea3781442da781f1d65f2e Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Wed, 20 Sep 2023 18:53:33 -0400 Subject: [PATCH 117/156] Add news --- news/12122.doc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12122.doc.rst diff --git a/news/12122.doc.rst b/news/12122.doc.rst new file mode 100644 index 000000000..49a3308a2 --- /dev/null +++ b/news/12122.doc.rst @@ -0,0 +1 @@ +Clarify --prefer-binary in CLI and docs From 3d6b0be901d5e8296b3d1303c70cbc38600f9cd6 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Thu, 21 Sep 2023 01:00:34 +0100 Subject: [PATCH 118/156] Remove outdated noqa comments --- news/80291DF4-7B0F-4268-B682-E1FCA1C3ACED.trivial.rst | 0 pyproject.toml | 1 + src/pip/_internal/cli/cmdoptions.py | 6 +++--- src/pip/_internal/cli/parser.py | 4 ++-- src/pip/_internal/commands/debug.py | 4 +--- src/pip/_internal/commands/install.py | 2 +- tests/functional/test_install_compat.py | 2 +- tests/functional/test_wheel.py | 2 +- 8 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 news/80291DF4-7B0F-4268-B682-E1FCA1C3ACED.trivial.rst diff --git a/news/80291DF4-7B0F-4268-B682-E1FCA1C3ACED.trivial.rst b/news/80291DF4-7B0F-4268-B682-E1FCA1C3ACED.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/pyproject.toml b/pyproject.toml index 7a4fe6246..b720c4602 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,7 @@ select = [ "PLE", "PLR0", "W", + "RUF100", ] [tool.ruff.isort] diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 84b40a8fc..8fb16dc4a 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -826,7 +826,7 @@ def _handle_config_settings( ) -> None: key, sep, val = value.partition("=") if sep != "=": - parser.error(f"Arguments to {opt_str} must be of the form KEY=VAL") # noqa + parser.error(f"Arguments to {opt_str} must be of the form KEY=VAL") dest = getattr(parser.values, option.dest) if dest is None: dest = {} @@ -921,13 +921,13 @@ def _handle_merge_hash( algo, digest = value.split(":", 1) except ValueError: parser.error( - "Arguments to {} must be a hash name " # noqa + "Arguments to {} must be a hash name " "followed by a value, like --hash=sha256:" "abcde...".format(opt_str) ) if algo not in STRONG_HASHES: parser.error( - "Allowed hash algorithms for {} are {}.".format( # noqa + "Allowed hash algorithms for {} are {}.".format( opt_str, ", ".join(STRONG_HASHES) ) ) diff --git a/src/pip/_internal/cli/parser.py b/src/pip/_internal/cli/parser.py index c762cf278..64cf97197 100644 --- a/src/pip/_internal/cli/parser.py +++ b/src/pip/_internal/cli/parser.py @@ -229,7 +229,7 @@ class ConfigOptionParser(CustomOptionParser): val = strtobool(val) except ValueError: self.error( - "{} is not a valid value for {} option, " # noqa + "{} is not a valid value for {} option, " "please specify a boolean value like yes/no, " "true/false or 1/0 instead.".format(val, key) ) @@ -240,7 +240,7 @@ class ConfigOptionParser(CustomOptionParser): val = int(val) if not isinstance(val, int) or val < 0: self.error( - "{} is not a valid value for {} option, " # noqa + "{} is not a valid value for {} option, " "please instead specify either a non-negative integer " "or a boolean value like yes/no or false/true " "which is equivalent to 1/0.".format(val, key) diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index 1b1fd3ea5..a29c625e8 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -134,9 +134,7 @@ def show_tags(options: Values) -> None: def ca_bundle_info(config: Configuration) -> str: - # Ruff misidentifies config as a dict. - # Configuration does not have support the mapping interface. - levels = {key.split(".", 1)[0] for key, _ in config.items()} # noqa: PERF102 + levels = {key.split(".", 1)[0] for key, _ in config.items()} if not levels: return "Not specified" diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index f6a300804..d88cafe5a 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -501,7 +501,7 @@ class InstallCommand(RequirementCommand): show_traceback, options.use_user_site, ) - logger.error(message, exc_info=show_traceback) # noqa + logger.error(message, exc_info=show_traceback) return ERROR diff --git a/tests/functional/test_install_compat.py b/tests/functional/test_install_compat.py index 8374d487b..6c809e753 100644 --- a/tests/functional/test_install_compat.py +++ b/tests/functional/test_install_compat.py @@ -11,7 +11,7 @@ from tests.lib import ( PipTestEnvironment, TestData, assert_all_changes, - pyversion, # noqa: F401 + pyversion, ) diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index c0e279492..042f58246 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -10,7 +10,7 @@ from pip._internal.cli.status_codes import ERROR from tests.lib import ( PipTestEnvironment, TestData, - pyversion, # noqa: F401 + pyversion, ) From eddd9ddb66e6a3b74d7d2f3187fd246c955e00b7 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 23 Sep 2023 09:10:13 -0700 Subject: [PATCH 119/156] Enable mypy's strict equality checks (#12209) This makes mypy check more behaviours within the codebase. Co-authored-by: Pradyun Gedam --- news/E2B261CA-A0CF-4309-B808-1210C0B54632.trivial.rst | 0 setup.cfg | 11 +++++++---- src/pip/_internal/utils/temp_dir.py | 4 ++-- tests/functional/test_list.py | 3 +-- tests/lib/__init__.py | 4 +++- tests/unit/test_network_auth.py | 2 +- tests/unit/test_options.py | 4 ++-- tests/unit/test_target_python.py | 10 +++++----- 8 files changed, 21 insertions(+), 17 deletions(-) create mode 100644 news/E2B261CA-A0CF-4309-B808-1210C0B54632.trivial.rst diff --git a/news/E2B261CA-A0CF-4309-B808-1210C0B54632.trivial.rst b/news/E2B261CA-A0CF-4309-B808-1210C0B54632.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index 2e35be30d..08a1795eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,11 +36,14 @@ per-file-ignores = [mypy] mypy_path = $MYPY_CONFIG_FILE_DIR/src + +strict = True + +no_implicit_reexport = False +allow_subclassing_any = True +allow_untyped_calls = True +warn_return_any = False ignore_missing_imports = True -disallow_untyped_defs = True -disallow_any_generics = True -warn_unused_ignores = True -no_implicit_optional = True [mypy-pip._internal.utils._jaraco_text] ignore_errors = True diff --git a/src/pip/_internal/utils/temp_dir.py b/src/pip/_internal/utils/temp_dir.py index 38c5f7c7c..4eec5f37f 100644 --- a/src/pip/_internal/utils/temp_dir.py +++ b/src/pip/_internal/utils/temp_dir.py @@ -6,9 +6,9 @@ import tempfile import traceback from contextlib import ExitStack, contextmanager from pathlib import Path -from types import FunctionType from typing import ( Any, + Callable, Dict, Generator, List, @@ -187,7 +187,7 @@ class TempDirectory: errors: List[BaseException] = [] def onerror( - func: FunctionType, + func: Callable[..., Any], path: Path, exc_val: BaseException, ) -> None: diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index cf8900a32..03dce41e7 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -595,8 +595,7 @@ def test_outdated_formats(script: PipTestEnvironment, data: TestData) -> None: "--outdated", "--format=json", ) - data = json.loads(result.stdout) - assert data == [ + assert json.loads(result.stdout) == [ { "name": "simple", "version": "1.0", diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index a48423570..2e8b239ac 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -46,7 +46,9 @@ if TYPE_CHECKING: # Literal was introduced in Python 3.8. from typing import Literal - ResolverVariant = Literal["resolvelib", "legacy"] + ResolverVariant = Literal[ + "resolvelib", "legacy", "2020-resolver", "legacy-resolver" + ] else: ResolverVariant = str diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index e3cb772bb..5bd85f8cd 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -352,7 +352,7 @@ class KeyringModuleV2: ), ) def test_keyring_get_credential( - monkeypatch: pytest.MonkeyPatch, url: str, expect: str + monkeypatch: pytest.MonkeyPatch, url: str, expect: Tuple[str, str] ) -> None: monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) auth = MultiDomainBasicAuth( diff --git a/tests/unit/test_options.py b/tests/unit/test_options.py index 43d5fdd3d..22ff7f721 100644 --- a/tests/unit/test_options.py +++ b/tests/unit/test_options.py @@ -2,7 +2,7 @@ import os from contextlib import contextmanager from optparse import Values from tempfile import NamedTemporaryFile -from typing import Any, Dict, Iterator, List, Tuple, Union, cast +from typing import Any, Dict, Iterator, List, Tuple, Type, Union, cast import pytest @@ -605,7 +605,7 @@ class TestOptionsConfigFiles: self, monkeypatch: pytest.MonkeyPatch, args: List[str], - expect: Union[None, str, PipError], + expect: Union[None, str, Type[PipError]], ) -> None: cmd = cast(ConfigurationCommand, create_command("config")) # Replace a handler with a no-op to avoid side effects diff --git a/tests/unit/test_target_python.py b/tests/unit/test_target_python.py index bc1713769..31df5935e 100644 --- a/tests/unit/test_target_python.py +++ b/tests/unit/test_target_python.py @@ -99,17 +99,17 @@ class TestTargetPython: py_version_info: Optional[Tuple[int, ...]], expected_version: Optional[str], ) -> None: - mock_get_supported.return_value = ["tag-1", "tag-2"] + dummy_tags = [Tag("py4", "none", "any"), Tag("py5", "none", "any")] + mock_get_supported.return_value = dummy_tags target_python = TargetPython(py_version_info=py_version_info) actual = target_python.get_sorted_tags() - assert actual == ["tag-1", "tag-2"] + assert actual == dummy_tags - actual = mock_get_supported.call_args[1]["version"] - assert actual == expected_version + assert mock_get_supported.call_args[1]["version"] == expected_version # Check that the value was cached. - assert target_python._valid_tags == ["tag-1", "tag-2"] + assert target_python._valid_tags == dummy_tags def test_get_unsorted_tags__uses_cached_value(self) -> None: """ From 666be3544b3a4f663f77399e5f82af55e72dbaae Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sat, 23 Sep 2023 13:33:18 -0700 Subject: [PATCH 120/156] Avoid use of 2020-resolver and legacy-resolver --- ...69-21F3-49F6-B938-AB16E326F82C.trivial.rst | 0 src/pip/_internal/cli/req_command.py | 6 +-- src/pip/_internal/commands/install.py | 4 +- tests/conftest.py | 4 +- tests/functional/test_freeze.py | 2 +- tests/functional/test_install.py | 8 ++-- tests/functional/test_install_extras.py | 2 +- tests/functional/test_install_reqs.py | 38 ++++++++----------- tests/functional/test_install_upgrade.py | 2 +- tests/lib/__init__.py | 4 +- tests/unit/test_req_file.py | 4 +- 11 files changed, 32 insertions(+), 42 deletions(-) create mode 100644 news/1F54AB69-21F3-49F6-B938-AB16E326F82C.trivial.rst diff --git a/news/1F54AB69-21F3-49F6-B938-AB16E326F82C.trivial.rst b/news/1F54AB69-21F3-49F6-B938-AB16E326F82C.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 86070f10c..96d8efaf5 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -268,7 +268,7 @@ class RequirementCommand(IndexGroupCommand): if "legacy-resolver" in options.deprecated_features_enabled: return "legacy" - return "2020-resolver" + return "resolvelib" @classmethod def make_requirement_preparer( @@ -290,7 +290,7 @@ class RequirementCommand(IndexGroupCommand): legacy_resolver = False resolver_variant = cls.determine_resolver_variant(options) - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": lazy_wheel = "fast-deps" in options.features_enabled if lazy_wheel: logger.warning( @@ -352,7 +352,7 @@ class RequirementCommand(IndexGroupCommand): # The long import name and duplicated invocation is needed to convince # Mypy into correctly typechecking. Otherwise it would complain the # "Resolver" class being redefined. - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": import pip._internal.resolution.resolvelib.resolver return pip._internal.resolution.resolvelib.resolver.Resolver( diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index f6a300804..d53bbd059 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -595,7 +595,7 @@ class InstallCommand(RequirementCommand): "source of the following dependency conflicts." ) else: - assert resolver_variant == "2020-resolver" + assert resolver_variant == "resolvelib" parts.append( "pip's dependency resolver does not currently take into account " "all the packages that are installed. This behaviour is the " @@ -628,7 +628,7 @@ class InstallCommand(RequirementCommand): requirement=req, dep_name=dep_name, dep_version=dep_version, - you=("you" if resolver_variant == "2020-resolver" else "you'll"), + you=("you" if resolver_variant == "resolvelib" else "you'll"), ) parts.append(message) diff --git a/tests/conftest.py b/tests/conftest.py index cd9931c66..07cd468c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,8 +76,8 @@ def pytest_addoption(parser: Parser) -> None: parser.addoption( "--resolver", action="store", - default="2020-resolver", - choices=["2020-resolver", "legacy"], + default="resolvelib", + choices=["resolvelib", "legacy"], help="use given resolver in tests", ) parser.addoption( diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index d6122308a..9a5937df3 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -629,7 +629,7 @@ _freeze_req_opts = textwrap.dedent( --extra-index-url http://ignore --find-links http://ignore --index-url http://ignore - --use-feature 2020-resolver + --use-feature resolvelib """ ) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index bf0512943..485710eaa 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1209,9 +1209,9 @@ def test_install_nonlocal_compatible_wheel_path( "--no-index", "--only-binary=:all:", Path(data.packages) / "simplewheel-2.0-py3-fakeabi-fakeplat.whl", - expect_error=(resolver_variant == "2020-resolver"), + expect_error=(resolver_variant == "resolvelib"), ) - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": assert result.returncode == ERROR else: assert result.returncode == SUCCESS @@ -1825,14 +1825,14 @@ def test_install_editable_with_wrong_egg_name( "install", "--editable", f"file://{pkga_path}#egg=pkgb", - expect_error=(resolver_variant == "2020-resolver"), + expect_error=(resolver_variant == "resolvelib"), ) assert ( "Generating metadata for package pkgb produced metadata " "for project name pkga. Fix your #egg=pkgb " "fragments." ) in result.stderr - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": assert "has inconsistent" in result.stdout, str(result) else: assert "Successfully installed pkga" in str(result), str(result) diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index 8ccbcf199..1dd67be0a 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -242,7 +242,7 @@ def test_install_extra_merging( expect_error=(fails_on_legacy and resolver_variant == "legacy"), ) - if not fails_on_legacy or resolver_variant == "2020-resolver": + if not fails_on_legacy or resolver_variant == "resolvelib": expected = f"Successfully installed pkga-0.1 simple-{simple_version}" assert expected in result.stdout diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index 96cff0dc5..c21b9ba83 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -392,11 +392,8 @@ def test_constraints_local_editable_install_causes_error( to_install, expect_error=True, ) - if resolver_variant == "legacy-resolver": - assert "Could not satisfy constraints" in result.stderr, str(result) - else: - # Because singlemodule only has 0.0.1 available. - assert "Cannot install singlemodule 0.0.1" in result.stderr, str(result) + # Because singlemodule only has 0.0.1 available. + assert "Cannot install singlemodule 0.0.1" in result.stderr, str(result) @pytest.mark.network @@ -426,11 +423,8 @@ def test_constraints_local_install_causes_error( to_install, expect_error=True, ) - if resolver_variant == "legacy-resolver": - assert "Could not satisfy constraints" in result.stderr, str(result) - else: - # Because singlemodule only has 0.0.1 available. - assert "Cannot install singlemodule 0.0.1" in result.stderr, str(result) + # Because singlemodule only has 0.0.1 available. + assert "Cannot install singlemodule 0.0.1" in result.stderr, str(result) def test_constraints_constrain_to_local_editable( @@ -451,9 +445,9 @@ def test_constraints_constrain_to_local_editable( script.scratch_path / "constraints.txt", "singlemodule", allow_stderr_warning=True, - expect_error=(resolver_variant == "2020-resolver"), + expect_error=(resolver_variant == "resolvelib"), ) - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": assert "Editable requirements are not allowed as constraints" in result.stderr else: assert "Running setup.py develop for singlemodule" in result.stdout @@ -551,9 +545,9 @@ def test_install_with_extras_from_constraints( script.scratch_path / "constraints.txt", "LocalExtras", allow_stderr_warning=True, - expect_error=(resolver_variant == "2020-resolver"), + expect_error=(resolver_variant == "resolvelib"), ) - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": assert "Constraints cannot have extras" in result.stderr else: result.did_create(script.site_packages / "simple") @@ -589,9 +583,9 @@ def test_install_with_extras_joined( script.scratch_path / "constraints.txt", "LocalExtras[baz]", allow_stderr_warning=True, - expect_error=(resolver_variant == "2020-resolver"), + expect_error=(resolver_variant == "resolvelib"), ) - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": assert "Constraints cannot have extras" in result.stderr else: result.did_create(script.site_packages / "simple") @@ -610,9 +604,9 @@ def test_install_with_extras_editable_joined( script.scratch_path / "constraints.txt", "LocalExtras[baz]", allow_stderr_warning=True, - expect_error=(resolver_variant == "2020-resolver"), + expect_error=(resolver_variant == "resolvelib"), ) - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": assert "Editable requirements are not allowed as constraints" in result.stderr else: result.did_create(script.site_packages / "simple") @@ -654,9 +648,9 @@ def test_install_distribution_union_with_constraints( script.scratch_path / "constraints.txt", f"{to_install}[baz]", allow_stderr_warning=True, - expect_error=(resolver_variant == "2020-resolver"), + expect_error=(resolver_variant == "resolvelib"), ) - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": msg = "Unnamed requirements are not allowed as constraints" assert msg in result.stderr else: @@ -674,9 +668,9 @@ def test_install_distribution_union_with_versions( result = script.pip_install_local( f"{to_install_001}[bar]", f"{to_install_002}[baz]", - expect_error=(resolver_variant == "2020-resolver"), + expect_error=(resolver_variant == "resolvelib"), ) - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": assert "Cannot install localextras[bar]" in result.stderr assert ("localextras[bar] 0.0.1 depends on localextras 0.0.1") in result.stdout assert ("localextras[baz] 0.0.2 depends on localextras 0.0.2") in result.stdout diff --git a/tests/functional/test_install_upgrade.py b/tests/functional/test_install_upgrade.py index 09c01d7eb..6556fcdf5 100644 --- a/tests/functional/test_install_upgrade.py +++ b/tests/functional/test_install_upgrade.py @@ -172,7 +172,7 @@ def test_upgrade_with_newest_already_installed( "install", "--upgrade", "-f", data.find_links, "--no-index", "simple" ) assert not result.files_created, "simple upgraded when it should not have" - if resolver_variant == "2020-resolver": + if resolver_variant == "resolvelib": msg = "Requirement already satisfied" else: msg = "already up-to-date" diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 2e8b239ac..a48423570 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -46,9 +46,7 @@ if TYPE_CHECKING: # Literal was introduced in Python 3.8. from typing import Literal - ResolverVariant = Literal[ - "resolvelib", "legacy", "2020-resolver", "legacy-resolver" - ] + ResolverVariant = Literal["resolvelib", "legacy"] else: ResolverVariant = str diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 439c41563..7a196eb8d 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -471,9 +471,7 @@ class TestProcessLine: ) -> None: """--use-feature triggers error when parsing requirements files.""" with pytest.raises(RequirementsFileParseError): - line_processor( - "--use-feature=2020-resolver", "filename", 1, options=options - ) + line_processor("--use-feature=resolvelib", "filename", 1, options=options) def test_relative_local_find_links( self, From ac19f79049dea400ae1c3ddd7937c1a3ba90f507 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sat, 23 Sep 2023 17:47:57 -0700 Subject: [PATCH 121/156] Follow imports for more vendored dependencies This will allow mypy to notice if you e.g. try to call a colorama function that does not exist. Note we won't report any errors in vendored code due to the ignore_errors config above. It would also be quite easy to let mypy look at pkg_resources code, but this would involve the addition of like three type ignores. --- news/12AE57EC-683C-4A8E-BCCB-851FCD0730B4.trivial.rst | 0 setup.cfg | 4 ---- 2 files changed, 4 deletions(-) create mode 100644 news/12AE57EC-683C-4A8E-BCCB-851FCD0730B4.trivial.rst diff --git a/news/12AE57EC-683C-4A8E-BCCB-851FCD0730B4.trivial.rst b/news/12AE57EC-683C-4A8E-BCCB-851FCD0730B4.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index 08a1795eb..b87fec7ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,12 +54,8 @@ ignore_errors = True # These vendored libraries use runtime magic to populate things and don't sit # well with static typing out of the box. Eventually we should provide correct # typing information for their public interface and remove these configs. -[mypy-pip._vendor.colorama] -follow_imports = skip [mypy-pip._vendor.pkg_resources] follow_imports = skip -[mypy-pip._vendor.progress.*] -follow_imports = skip [mypy-pip._vendor.requests.*] follow_imports = skip From 64d2dc3253e4a81e437931fb9b30d636556461d1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Sep 2023 10:28:27 -0400 Subject: [PATCH 122/156] Fix lints --- src/pip/_internal/commands/cache.py | 8 ++++---- tests/functional/test_cache.py | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 32d1a221d..1f3b5fe14 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -96,9 +96,9 @@ class CacheCommand(Command): http_cache_location = self._cache_dir(options, "http-v2") old_http_cache_location = self._cache_dir(options, "http") wheels_cache_location = self._cache_dir(options, "wheels") - http_cache_size = ( - filesystem.format_size(filesystem.directory_size(http_cache_location) + - filesystem.directory_size(old_http_cache_location)) + http_cache_size = filesystem.format_size( + filesystem.directory_size(http_cache_location) + + filesystem.directory_size(old_http_cache_location) ) wheels_cache_size = filesystem.format_directory_size(wheels_cache_location) @@ -112,7 +112,7 @@ class CacheCommand(Command): Locally built wheels location: {wheels_cache_location} Locally built wheels size: {wheels_cache_size} Number of locally built wheels: {package_count} - """ + """ # noqa: E501 ) .format( http_cache_location=http_cache_location, diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index c5d910d45..a744dbbb9 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -203,7 +203,10 @@ def test_cache_info( ) -> None: result = script.pip("cache", "info") - assert f"Package index page cache location (pip v23.3+): {http_cache_dir}" in result.stdout + assert ( + f"Package index page cache location (pip v23.3+): {http_cache_dir}" + in result.stdout + ) assert f"Locally built wheels location: {wheel_cache_dir}" in result.stdout num_wheels = len(wheel_cache_files) assert f"Number of locally built wheels: {num_wheels}" in result.stdout From 9692d48822187d3b0107cc4b1333d74cf3374222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 1 Oct 2023 11:51:37 +0200 Subject: [PATCH 123/156] Drop isort and flake8 settings from setup.cfg Since we use ruff, these are not used anymore. --- setup.cfg | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/setup.cfg b/setup.cfg index b87fec7ef..0be3ef08b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,39 +1,3 @@ -[isort] -profile = black -skip = - ./build, - .nox, - .tox, - .scratch, - _vendor, - data -known_third_party = - pip._vendor - -[flake8] -max-line-length = 88 -exclude = - ./build, - .nox, - .tox, - .scratch, - _vendor, - data -enable-extensions = G -extend-ignore = - G200, G202, - # black adds spaces around ':' - E203, - # using a cache - B019, - # reassigning variables in a loop - B020, -per-file-ignores = - # G: The plugin logging-format treats every .log and .error as logging. - noxfile.py: G - # B011: Do not call assert False since python -O removes these calls - tests/*: B011 - [mypy] mypy_path = $MYPY_CONFIG_FILE_DIR/src From 3f6e81694f8cce0866d934e1deedbdccdd0deb6b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 1 Oct 2023 12:22:41 +0100 Subject: [PATCH 124/156] Rework how the logging stack handles rich objects This makes it possible to render content via rich without a magic string and relies on a proper mechanism supported by the logging stack. --- src/pip/_internal/cli/base_command.py | 2 +- src/pip/_internal/self_outdated_check.py | 2 +- src/pip/_internal/utils/logging.py | 4 ++-- src/pip/_internal/utils/subprocess.py | 2 +- tests/unit/test_utils_subprocess.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 6a3b8e6c2..db9d5cc66 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -181,7 +181,7 @@ class Command(CommandContextMixIn): assert isinstance(status, int) return status except DiagnosticPipError as exc: - logger.error("[present-rich] %s", exc) + logger.error("%s", exc, extra={"rich": True}) logger.debug("Exception information:", exc_info=True) return ERROR diff --git a/src/pip/_internal/self_outdated_check.py b/src/pip/_internal/self_outdated_check.py index eefbc498b..cb18edbed 100644 --- a/src/pip/_internal/self_outdated_check.py +++ b/src/pip/_internal/self_outdated_check.py @@ -233,7 +233,7 @@ def pip_self_version_check(session: PipSession, options: optparse.Values) -> Non ), ) if upgrade_prompt is not None: - logger.warning("[present-rich] %s", upgrade_prompt) + logger.warning("%s", upgrade_prompt, extra={"rich": True}) except Exception: logger.warning("There was an error checking the latest version of pip.") logger.debug("See below for error", exc_info=True) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index c10e1f4ce..95982dfb6 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -155,8 +155,8 @@ class RichPipStreamHandler(RichHandler): # If we are given a diagnostic error to present, present it with indentation. assert isinstance(record.args, tuple) - if record.msg == "[present-rich] %s" and len(record.args) == 1: - rich_renderable = record.args[0] + if getattr(record, "rich", False): + (rich_renderable,) = record.args assert isinstance( rich_renderable, (ConsoleRenderable, RichCast, str) ), f"{rich_renderable} is not rich-console-renderable" diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py index 1e8ff50ed..79580b053 100644 --- a/src/pip/_internal/utils/subprocess.py +++ b/src/pip/_internal/utils/subprocess.py @@ -209,7 +209,7 @@ def call_subprocess( output_lines=all_output if not showing_subprocess else None, ) if log_failed_cmd: - subprocess_logger.error("[present-rich] %s", error) + subprocess_logger.error("%s", error, extra={"rich": True}) subprocess_logger.verbose( "[bold magenta]full command[/]: [blue]%s[/]", escape(format_command_args(cmd)), diff --git a/tests/unit/test_utils_subprocess.py b/tests/unit/test_utils_subprocess.py index a694b717f..2dbd5d77e 100644 --- a/tests/unit/test_utils_subprocess.py +++ b/tests/unit/test_utils_subprocess.py @@ -260,9 +260,9 @@ class TestCallSubprocess: expected = ( None, [ - # pytest's caplog overrides th formatter, which means that we + # pytest's caplog overrides the formatter, which means that we # won't see the message formatted through our formatters. - ("pip.subprocessor", ERROR, "[present-rich]"), + ("pip.subprocessor", ERROR, "subprocess error exited with 1"), ], ) # The spinner should spin three times in this case since the From ccc4bbcdfd06b1903af3b4cf6bf845be3b3c8b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 1 Oct 2023 15:05:20 +0200 Subject: [PATCH 125/156] Postpone some deprecation removals --- src/pip/_internal/req/req_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index f8957e5d9..dd8a0db27 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -514,7 +514,7 @@ class InstallRequirement: "to use --use-pep517 or add a " "pyproject.toml file to the project" ), - gone_in="23.3", + gone_in="24.0", ) self.use_pep517 = False return @@ -904,7 +904,7 @@ def check_legacy_setup_py_options( reason="--build-option and --global-option are deprecated.", issue=11859, replacement="to use --config-settings", - gone_in="23.3", + gone_in="24.0", ) logger.warning( "Implying --no-binary=:all: due to the presence of " From 389cb799d0da9a840749fcd14878928467ed49b4 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 1 Oct 2023 14:10:25 +0100 Subject: [PATCH 126/156] Use `-r=...` instead of `-r ...` for hg This ensures that the resulting revision can not be misinterpreted as an option. --- src/pip/_internal/vcs/mercurial.py | 2 +- tests/unit/test_vcs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 4595960b5..e440c1221 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -31,7 +31,7 @@ class Mercurial(VersionControl): @staticmethod def get_base_rev_args(rev: str) -> List[str]: - return ["-r", rev] + return [f"-r={rev}"] def fetch_new( self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index fb6c3ea31..4a3750f2d 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -66,7 +66,7 @@ def test_rev_options_repr() -> None: # First check VCS-specific RevOptions behavior. (Bazaar, [], ["-r", "123"], {}), (Git, ["HEAD"], ["123"], {}), - (Mercurial, [], ["-r", "123"], {}), + (Mercurial, [], ["-r=123"], {}), (Subversion, [], ["-r", "123"], {}), # Test extra_args. For this, test using a single VersionControl class. ( From 408b5248dc8934af50811190cb7df913116031b0 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 1 Oct 2023 13:49:06 +0100 Subject: [PATCH 127/156] :newspaper: --- news/12306.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12306.bugfix.rst diff --git a/news/12306.bugfix.rst b/news/12306.bugfix.rst new file mode 100644 index 000000000..eb6eecaaf --- /dev/null +++ b/news/12306.bugfix.rst @@ -0,0 +1 @@ +Use ``-r=...`` instead of ``-r ...`` to specify references with Mercurial. From dcb9dc03698b8c59451b98d236ae44b4959c0343 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 3 Oct 2023 09:01:40 +0200 Subject: [PATCH 128/156] Wrap long lines --- .pre-commit-config.yaml | 3 +-- tests/conftest.py | 5 ++++- tests/unit/test_collector.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c8d81deed..2c576d90a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,8 +22,7 @@ repos: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.0.287 + rev: v0.0.292 hooks: - id: ruff diff --git a/tests/conftest.py b/tests/conftest.py index 07cd468c1..62191f99c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1031,7 +1031,10 @@ def html_index_with_onetime_server( class InDirectoryServer(http.server.ThreadingHTTPServer): def finish_request(self, request: Any, client_address: Any) -> None: self.RequestHandlerClass( - request, client_address, self, directory=str(html_index_for_packages) # type: ignore[call-arg] # noqa: E501 + request, + client_address, + self, + directory=str(html_index_for_packages), # type: ignore[call-arg] ) class Handler(OneTimeDownloadHandler): diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index 5410a4afc..3c8b81de4 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -625,7 +625,7 @@ _pkg1_requirement = Requirement("pkg1==1.0") ), # Test with a provided hash value. ( - '', # noqa: E501 + '', MetadataFile({"sha256": "aa113592bbe"}), {}, ), From ac962890b513253376f543febbe189a1aca26ef9 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Wed, 24 May 2023 20:43:20 -0500 Subject: [PATCH 129/156] Add a dependabot config to update CI actions monthly --- .github/dependabot.yml | 6 ++++++ news/d7179b28-bc23-46aa-9175-834117a42dbd.trivial.rst | 0 2 files changed, 6 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 news/d7179b28-bc23-46aa-9175-834117a42dbd.trivial.rst diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..8ac6b8c49 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" diff --git a/news/d7179b28-bc23-46aa-9175-834117a42dbd.trivial.rst b/news/d7179b28-bc23-46aa-9175-834117a42dbd.trivial.rst new file mode 100644 index 000000000..e69de29bb From dba399fe6a41615d0a59899f7ac6dfdd26a42731 Mon Sep 17 00:00:00 2001 From: Wu Zhenyu Date: Sat, 22 Jul 2023 15:31:12 +0800 Subject: [PATCH 130/156] Fix #12166 - tests expected results indendation was off - add bugfix news entry --- news/12166.bugfix.rst | 1 + src/pip/_internal/commands/completion.py | 15 ++++++++++++--- tests/functional/test_completion.py | 15 ++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 news/12166.bugfix.rst diff --git a/news/12166.bugfix.rst b/news/12166.bugfix.rst new file mode 100644 index 000000000..491597c7f --- /dev/null +++ b/news/12166.bugfix.rst @@ -0,0 +1 @@ +Fix completion script for zsh diff --git a/src/pip/_internal/commands/completion.py b/src/pip/_internal/commands/completion.py index 30233fc7a..9e89e2798 100644 --- a/src/pip/_internal/commands/completion.py +++ b/src/pip/_internal/commands/completion.py @@ -23,9 +23,18 @@ COMPLETION_SCRIPTS = { """, "zsh": """ #compdef -P pip[0-9.]# - compadd $( COMP_WORDS="$words[*]" \\ - COMP_CWORD=$((CURRENT-1)) \\ - PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null ) + __pip() {{ + compadd $( COMP_WORDS="$words[*]" \\ + COMP_CWORD=$((CURRENT-1)) \\ + PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null ) + }} + if [[ $zsh_eval_context[-1] == loadautofunc ]]; then + # autoload from fpath, call function directly + __pip "$@" + else + # eval/source/. command, register function for later + compdef __pip -P 'pip[0-9.]#' + fi """, "fish": """ function __fish_complete_pip diff --git a/tests/functional/test_completion.py b/tests/functional/test_completion.py index 2e3f31729..2aa861aac 100644 --- a/tests/functional/test_completion.py +++ b/tests/functional/test_completion.py @@ -44,9 +44,18 @@ complete -fa "(__fish_complete_pip)" -c pip""", "zsh", """\ #compdef -P pip[0-9.]# -compadd $( COMP_WORDS="$words[*]" \\ - COMP_CWORD=$((CURRENT-1)) \\ - PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )""", +__pip() { + compadd $( COMP_WORDS="$words[*]" \\ + COMP_CWORD=$((CURRENT-1)) \\ + PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null ) +} +if [[ $zsh_eval_context[-1] == loadautofunc ]]; then + # autoload from fpath, call function directly + __pip "$@" +else + # eval/source/. command, register function for later + compdef __pip -P 'pip[0-9.]#' +fi""", ), ( "powershell", From 431cf5af82f43431b75fa495b114c1177ab97f8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 19:00:48 +0000 Subject: [PATCH 131/156] Bump dessant/lock-threads from 3 to 4 Bumps [dessant/lock-threads](https://github.com/dessant/lock-threads) from 3 to 4. - [Release notes](https://github.com/dessant/lock-threads/releases) - [Changelog](https://github.com/dessant/lock-threads/blob/main/CHANGELOG.md) - [Commits](https://github.com/dessant/lock-threads/compare/v3...v4) --- updated-dependencies: - dependency-name: dessant/lock-threads dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lock-threads.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index 990440dd6..dc68b683b 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -17,7 +17,7 @@ jobs: if: github.repository_owner == 'pypa' runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v3 + - uses: dessant/lock-threads@v4 with: issue-inactive-days: '30' pr-inactive-days: '15' From 0042cc94cc6d9b74307343107b3812fb9c56fda8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 19:00:51 +0000 Subject: [PATCH 132/156] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/news-file.yml | 2 +- .github/workflows/update-rtd-redirects.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41d3ab946..5f7cd942b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.x" @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.x" @@ -81,7 +81,7 @@ jobs: github.event_name != 'pull_request' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.x" @@ -112,7 +112,7 @@ jobs: - "3.12" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -164,7 +164,7 @@ jobs: group: [1, 2] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -215,7 +215,7 @@ jobs: github.event_name != 'pull_request' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.github/workflows/news-file.yml b/.github/workflows/news-file.yml index 371e12fd7..398ad1b7e 100644 --- a/.github/workflows/news-file.yml +++ b/.github/workflows/news-file.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # `towncrier check` runs `git diff --name-only origin/main...`, which # needs a non-shallow clone. diff --git a/.github/workflows/update-rtd-redirects.yml b/.github/workflows/update-rtd-redirects.yml index 8259b6c0b..c333a09a3 100644 --- a/.github/workflows/update-rtd-redirects.yml +++ b/.github/workflows/update-rtd-redirects.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest environment: RTD Deploys steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.11" From d9b47d0173b5d531a1c5a10172c73c37f43d8f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 8 Oct 2023 17:19:09 +0200 Subject: [PATCH 133/156] Update egg deprecation message --- src/pip/_internal/metadata/importlib/_envs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/metadata/importlib/_envs.py b/src/pip/_internal/metadata/importlib/_envs.py index 3850ddaf4..048dc55dc 100644 --- a/src/pip/_internal/metadata/importlib/_envs.py +++ b/src/pip/_internal/metadata/importlib/_envs.py @@ -151,7 +151,8 @@ def _emit_egg_deprecation(location: Optional[str]) -> None: deprecated( reason=f"Loading egg at {location} is deprecated.", replacement="to use pip for package installation.", - gone_in="23.3", + gone_in="24.3", + issue=12330, ) From 76a8c0f2652027b4938a13e9abef500c5f43b185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 8 Oct 2023 18:17:05 +0200 Subject: [PATCH 134/156] Postpone deprecation of legacy versions and specifiers --- src/pip/_internal/operations/check.py | 4 ++-- src/pip/_internal/req/req_set.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index 261045922..1b7fd7ab7 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -168,7 +168,7 @@ def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None: f"release a version with a conforming version number" ), issue=12063, - gone_in="23.3", + gone_in="24.0", ) for dep in package_details.dependencies: if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier): @@ -183,5 +183,5 @@ def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None: f"release a version with a conforming dependency specifiers" ), issue=12063, - gone_in="23.3", + gone_in="24.0", ) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index cff676017..1bf73d595 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -99,7 +99,7 @@ class RequirementSet: "or contact the package author to fix the version number" ), issue=12063, - gone_in="23.3", + gone_in="24.0", ) for dep in req.get_dist().iter_dependencies(): if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier): @@ -115,5 +115,5 @@ class RequirementSet: "or contact the package author to fix the version number" ), issue=12063, - gone_in="23.3", + gone_in="24.0", ) From 496b268c1b9ce3466c08eb4819e5460a943d1793 Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Wed, 11 Oct 2023 11:36:40 -0400 Subject: [PATCH 135/156] Update "Running Tests" documentation (#12334) Co-authored-by: Paul Moore Co-authored-by: Pradyun Gedam --- docs/html/development/getting-started.rst | 14 +++++++++++++- news/12334.doc.rst | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 news/12334.doc.rst diff --git a/docs/html/development/getting-started.rst b/docs/html/development/getting-started.rst index e248259f0..34d647fc2 100644 --- a/docs/html/development/getting-started.rst +++ b/docs/html/development/getting-started.rst @@ -73,7 +73,7 @@ pip's tests are written using the :pypi:`pytest` test framework and :mod:`unittest.mock`. :pypi:`nox` is used to automate the setup and execution of pip's tests. -It is preferable to run the tests in parallel for better experience during development, +It is preferable to run the tests in parallel for a better experience during development, since the tests can take a long time to finish when run sequentially. To run tests: @@ -104,6 +104,15 @@ can select tests using the various ways that pytest provides: $ # Using keywords $ nox -s test-3.10 -- -k "install and not wheel" +.. note:: + + When running pip's tests with OS distribution Python versions, be aware that some + functional tests may fail due to potential patches introduced by the distribution. + For all tests to pass consider: + + - Installing Python from `python.org`_ or compile from source + - Or, using `pyenv`_ to assist with source compilation + Running pip's entire test suite requires supported version control tools (subversion, bazaar, git, and mercurial) to be installed. If you are missing any of these VCS, those tests should be skipped automatically. You can also @@ -114,6 +123,9 @@ explicitly tell pytest to skip those tests: $ nox -s test-3.10 -- -k "not svn" $ nox -s test-3.10 -- -k "not (svn or git)" +.. _python.org: https://www.python.org/downloads/ +.. _pyenv: https://github.com/pyenv/pyenv + Running Linters =============== diff --git a/news/12334.doc.rst b/news/12334.doc.rst new file mode 100644 index 000000000..ff3d877e5 --- /dev/null +++ b/news/12334.doc.rst @@ -0,0 +1 @@ +Document that using OS-provided Python can cause pip's test suite to report false failures. From 2333ef3b53a71fb7acc9e76d6ff90409576b2250 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 12 Oct 2023 12:12:06 +0100 Subject: [PATCH 136/156] Upgrade urllib3 to 1.26.17 (#12343) --- news/urllib3.vendor.rst | 1 + src/pip/_vendor/urllib3/_version.py | 2 +- src/pip/_vendor/urllib3/request.py | 21 +++++++++++++++++++++ src/pip/_vendor/urllib3/util/retry.py | 2 +- src/pip/_vendor/vendor.txt | 2 +- 5 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 news/urllib3.vendor.rst diff --git a/news/urllib3.vendor.rst b/news/urllib3.vendor.rst new file mode 100644 index 000000000..37032f67a --- /dev/null +++ b/news/urllib3.vendor.rst @@ -0,0 +1 @@ +Upgrade urllib3 to 1.26.17 diff --git a/src/pip/_vendor/urllib3/_version.py b/src/pip/_vendor/urllib3/_version.py index d69ca3145..cad75fb5d 100644 --- a/src/pip/_vendor/urllib3/_version.py +++ b/src/pip/_vendor/urllib3/_version.py @@ -1,2 +1,2 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.16" +__version__ = "1.26.17" diff --git a/src/pip/_vendor/urllib3/request.py b/src/pip/_vendor/urllib3/request.py index 398386a5b..3b4cf9992 100644 --- a/src/pip/_vendor/urllib3/request.py +++ b/src/pip/_vendor/urllib3/request.py @@ -1,6 +1,9 @@ from __future__ import absolute_import +import sys + from .filepost import encode_multipart_formdata +from .packages import six from .packages.six.moves.urllib.parse import urlencode __all__ = ["RequestMethods"] @@ -168,3 +171,21 @@ class RequestMethods(object): extra_kw.update(urlopen_kw) return self.urlopen(method, url, **extra_kw) + + +if not six.PY2: + + class RequestModule(sys.modules[__name__].__class__): + def __call__(self, *args, **kwargs): + """ + If user tries to call this module directly urllib3 v2.x style raise an error to the user + suggesting they may need urllib3 v2 + """ + raise TypeError( + "'module' object is not callable\n" + "urllib3.request() method is not supported in this release, " + "upgrade to urllib3 v2 to use it\n" + "see https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html" + ) + + sys.modules[__name__].__class__ = RequestModule diff --git a/src/pip/_vendor/urllib3/util/retry.py b/src/pip/_vendor/urllib3/util/retry.py index 2490d5e5b..60ef6c4f3 100644 --- a/src/pip/_vendor/urllib3/util/retry.py +++ b/src/pip/_vendor/urllib3/util/retry.py @@ -235,7 +235,7 @@ class Retry(object): RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) #: Default headers to be used for ``remove_headers_on_redirect`` - DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) #: Maximum backoff time. DEFAULT_BACKOFF_MAX = 120 diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 43ced2a4b..8dbe13413 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -11,7 +11,7 @@ requests==2.31.0 certifi==2023.7.22 chardet==5.1.0 idna==3.4 - urllib3==1.26.16 + urllib3==1.26.17 rich==13.4.2 pygments==2.15.1 typing_extensions==4.7.1 From d1659b87e46abd0a2dcc74f2160dd52e6190e13b Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 10 Oct 2023 21:49:43 +0100 Subject: [PATCH 137/156] Correct issue number for NEWS entry added by #12197 The NEWS entry added in PR #12197 referenced issue #12191, however, the issue it actually fixed was #11847. --- news/{12191.bugfix.rst => 11847.bugfix.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{12191.bugfix.rst => 11847.bugfix.rst} (100%) diff --git a/news/12191.bugfix.rst b/news/11847.bugfix.rst similarity index 100% rename from news/12191.bugfix.rst rename to news/11847.bugfix.rst From 8f0ed32413daa411a728b50cd7776b9c02b010d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 14 Oct 2023 13:50:49 +0200 Subject: [PATCH 138/156] Redact URLs in Collecting... logs --- news/12350.bugfix.rst | 1 + src/pip/_internal/operations/prepare.py | 3 ++- src/pip/_internal/req/req_install.py | 3 ++- src/pip/_internal/utils/misc.py | 8 ++++++++ tests/unit/test_utils.py | 26 +++++++++++++++++++++++++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 news/12350.bugfix.rst diff --git a/news/12350.bugfix.rst b/news/12350.bugfix.rst new file mode 100644 index 000000000..3fb16b4ed --- /dev/null +++ b/news/12350.bugfix.rst @@ -0,0 +1 @@ +Redact password from URLs in some additional places. diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 1b32d7eec..488e76358 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -47,6 +47,7 @@ from pip._internal.utils.misc import ( display_path, hash_file, hide_url, + redact_auth_from_requirement, ) from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.unpacking import unpack_file @@ -277,7 +278,7 @@ class RequirementPreparer: information = str(display_path(req.link.file_path)) else: message = "Collecting %s" - information = str(req.req or req) + information = redact_auth_from_requirement(req.req) if req.req else str(req) # If we used req.req, inject requirement source if available (this # would already be included if we used req directly) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index dd8a0db27..e556be2b4 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -49,6 +49,7 @@ from pip._internal.utils.misc import ( display_path, hide_url, is_installable_dir, + redact_auth_from_requirement, redact_auth_from_url, ) from pip._internal.utils.packaging import safe_extra @@ -188,7 +189,7 @@ class InstallRequirement: def __str__(self) -> str: if self.req: - s = str(self.req) + s = redact_auth_from_requirement(self.req) if self.link: s += " from {}".format(redact_auth_from_url(self.link.url)) elif self.link: diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 9a6353fc8..78060e864 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -35,6 +35,7 @@ from typing import ( cast, ) +from pip._vendor.packaging.requirements import Requirement from pip._vendor.pyproject_hooks import BuildBackendHookCaller from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed @@ -578,6 +579,13 @@ def redact_auth_from_url(url: str) -> str: return _transform_url(url, _redact_netloc)[0] +def redact_auth_from_requirement(req: Requirement) -> str: + """Replace the password in a given requirement url with ****.""" + if not req.url: + return str(req) + return str(req).replace(req.url, redact_auth_from_url(req.url)) + + class HiddenText: def __init__(self, secret: str, redacted: str) -> None: self.secret = secret diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index d3b0d32d1..1352b7664 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -14,6 +14,7 @@ from typing import Any, Callable, Iterator, List, NoReturn, Optional, Tuple, Typ from unittest.mock import Mock, patch import pytest +from pip._vendor.packaging.requirements import Requirement from pip._internal.exceptions import HashMismatch, HashMissing, InstallationError from pip._internal.utils.deprecation import PipDeprecationWarning, deprecated @@ -37,6 +38,7 @@ from pip._internal.utils.misc import ( normalize_path, normalize_version_info, parse_netloc, + redact_auth_from_requirement, redact_auth_from_url, redact_netloc, remove_auth_from_url, @@ -765,6 +767,30 @@ def test_redact_auth_from_url(auth_url: str, expected_url: str) -> None: assert url == expected_url +@pytest.mark.parametrize( + "req, expected", + [ + ("pkga", "pkga"), + ( + "resolvelib@ " + " git+https://test-user:test-pass@github.com/sarugaku/resolvelib@1.0.1", + "resolvelib@" + " git+https://test-user:****@github.com/sarugaku/resolvelib@1.0.1", + ), + ( + "resolvelib@" + " git+https://test-user:test-pass@github.com/sarugaku/resolvelib@1.0.1" + " ; python_version>='3.6'", + "resolvelib@" + " git+https://test-user:****@github.com/sarugaku/resolvelib@1.0.1" + ' ; python_version >= "3.6"', + ), + ], +) +def test_redact_auth_from_requirement(req: str, expected: str) -> None: + assert redact_auth_from_requirement(Requirement(req)) == expected + + class TestHiddenText: def test_basic(self) -> None: """ From 8ff33edfc5824472c275be22bb0ea64c335fe651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 10:08:24 +0200 Subject: [PATCH 139/156] Don't mention setuptools in release process docs --- docs/html/development/release-process.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst index b71e2820b..65ed56707 100644 --- a/docs/html/development/release-process.rst +++ b/docs/html/development/release-process.rst @@ -145,8 +145,8 @@ Creating a new release #. Push the tag created by ``prepare-release``. #. Regenerate the ``get-pip.py`` script in the `get-pip repository`_ (as documented there) and commit the results. -#. Submit a Pull Request to `CPython`_ adding the new version of pip (and upgrading - setuptools) to ``Lib/ensurepip/_bundled``, removing the existing version, and +#. Submit a Pull Request to `CPython`_ adding the new version of pip + to ``Lib/ensurepip/_bundled``, removing the existing version, and adjusting the versions listed in ``Lib/ensurepip/__init__.py``. From bf9a9cbdae96ab99dd167a7c212c6076eceb7128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 10:20:24 +0200 Subject: [PATCH 140/156] Mention 'skip news' label in docs --- docs/html/development/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/development/contributing.rst b/docs/html/development/contributing.rst index 87734ee4d..b2f6f1d13 100644 --- a/docs/html/development/contributing.rst +++ b/docs/html/development/contributing.rst @@ -112,7 +112,7 @@ the ``news/`` directory with the extension of ``.trivial.rst``. If you are on a POSIX like operating system, one can be added by running ``touch news/$(uuidgen).trivial.rst``. On Windows, the same result can be achieved in Powershell using ``New-Item "news/$([guid]::NewGuid()).trivial.rst"``. -Core committers may also add a "trivial" label to the PR which will accomplish +Core committers may also add a "skip news" label to the PR which will accomplish the same thing. Upgrading, removing, or adding a new vendored library gets a special mention From 8d0278771c7325b04f02cb073c8ef02827cbeb93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 10:22:52 +0200 Subject: [PATCH 141/156] Reclassify news fragment This is not for the process category, and probably not significant enough for a feature news entry. --- news/{12155.process.rst => 12155.trivial.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{12155.process.rst => 12155.trivial.rst} (100%) diff --git a/news/12155.process.rst b/news/12155.trivial.rst similarity index 100% rename from news/12155.process.rst rename to news/12155.trivial.rst From 3e85558b10722598fb3353126e2f19979f7cf7dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 10:23:01 +0200 Subject: [PATCH 142/156] Update AUTHORS.txt --- AUTHORS.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/AUTHORS.txt b/AUTHORS.txt index 77eb39a42..49e30f696 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -20,6 +20,7 @@ Albert-Guan albertg Alberto Sottile Aleks Bunin +Ales Erjavec Alethea Flowers Alex Gaynor Alex Grönholm @@ -30,6 +31,7 @@ Alex Stachowiak Alexander Shtyrov Alexandre Conrad Alexey Popravka +Aleš Erjavec Alli Ami Fischman Ananya Maiti @@ -196,9 +198,11 @@ David Runge David Tucker David Wales Davidovich +ddelange Deepak Sharma Deepyaman Datta Denise Yu +dependabot[bot] derwolfe Desetude Devesh Kumar Singh @@ -312,6 +316,7 @@ Ilya Baryshev Inada Naoki Ionel Cristian Mărieș Ionel Maries Cristian +Itamar Turner-Trauring Ivan Pozdeev Jacob Kim Jacob Walls @@ -338,6 +343,7 @@ Jay Graves Jean-Christophe Fillion-Robin Jeff Barber Jeff Dairiki +Jeff Widman Jelmer Vernooij jenix21 Jeremy Stanley @@ -367,6 +373,7 @@ Joseph Long Josh Bronson Josh Hansen Josh Schneier +Joshua Juan Luis Cano Rodríguez Juanjo Bazán Judah Rand @@ -397,6 +404,7 @@ KOLANICH kpinc Krishna Oza Kumar McMillan +Kurt McKee Kyle Persohn lakshmanaram Laszlo Kiss-Kollar @@ -413,6 +421,7 @@ lorddavidiii Loren Carvalho Lucas Cimon Ludovic Gasc +Lukas Geiger Lukas Juhrich Luke Macken Luo Jiebin @@ -529,6 +538,7 @@ Patrick Jenkins Patrick Lawson patricktokeeffe Patrik Kopkan +Paul Ganssle Paul Kehrer Paul Moore Paul Nasrat @@ -609,6 +619,7 @@ ryneeverett Sachi King Salvatore Rinchiera sandeepkiran-js +Sander Van Balen Savio Jomton schlamar Scott Kitterman @@ -621,6 +632,7 @@ SeongSoo Cho Sergey Vasilyev Seth Michael Larson Seth Woodworth +Shahar Epstein Shantanu shireenrao Shivansh-007 @@ -648,6 +660,7 @@ Steve Kowalik Steven Myint Steven Silvester stonebig +studioj Stéphane Bidoul Stéphane Bidoul (ACSONE) Stéphane Klein From e3dc91dad93a020b3034a87ebe59027f63370fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 10:23:02 +0200 Subject: [PATCH 143/156] Bump for release --- NEWS.rst | 57 +++++++++++++++++++ news/11394.bugfix.rst | 1 - news/11649.bugfix.rst | 5 -- news/11847.bugfix.rst | 1 - news/11924.bugfix.rst | 1 - news/11924.feature.rst | 1 - news/12005.bugfix.rst | 1 - news/12059.doc.rst | 1 - news/12095.bugfix.rst | 1 - news/12122.doc.rst | 1 - news/12155.trivial.rst | 6 -- news/12166.bugfix.rst | 1 - news/12175.removal.rst | 1 - news/12183.trivial.rst | 1 - news/12187.bugfix.rst | 1 - news/12194.trivial.rst | 1 - news/12204.feature.rst | 1 - news/12215.feature.rst | 1 - news/12224.feature.rst | 1 - news/12225.bugfix.rst | 1 - news/12252.trivial.rst | 0 news/12254.process.rst | 1 - news/12261.trivial.rst | 0 news/12280.bugfix.rst | 1 - news/12306.bugfix.rst | 1 - news/12334.doc.rst | 1 - news/12350.bugfix.rst | 1 - ...EC-683C-4A8E-BCCB-851FCD0730B4.trivial.rst | 0 ...69-21F3-49F6-B938-AB16E326F82C.trivial.rst | 0 news/2984.bugfix.rst | 1 - ...FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst | 1 - ...DE-8011-4146-8CAD-85D7756D88A6.trivial.rst | 0 ...F4-7B0F-4268-B682-E1FCA1C3ACED.trivial.rst | 0 ...60-68FF-4C1E-A2CB-CF8634829D2D.trivial.rst | 0 ...CA-A0CF-4309-B808-1210C0B54632.trivial.rst | 0 news/certifi.vendor.rst | 1 - ...28-bc23-46aa-9175-834117a42dbd.trivial.rst | 0 news/truststore.vendor.rst | 1 - news/urllib3.vendor.rst | 1 - news/zhsdgdlsjgksdfj.trivial.rst | 0 src/pip/__init__.py | 2 +- 41 files changed, 58 insertions(+), 39 deletions(-) delete mode 100644 news/11394.bugfix.rst delete mode 100644 news/11649.bugfix.rst delete mode 100644 news/11847.bugfix.rst delete mode 100644 news/11924.bugfix.rst delete mode 100644 news/11924.feature.rst delete mode 100644 news/12005.bugfix.rst delete mode 100644 news/12059.doc.rst delete mode 100644 news/12095.bugfix.rst delete mode 100644 news/12122.doc.rst delete mode 100644 news/12155.trivial.rst delete mode 100644 news/12166.bugfix.rst delete mode 100644 news/12175.removal.rst delete mode 100644 news/12183.trivial.rst delete mode 100644 news/12187.bugfix.rst delete mode 100644 news/12194.trivial.rst delete mode 100644 news/12204.feature.rst delete mode 100644 news/12215.feature.rst delete mode 100644 news/12224.feature.rst delete mode 100644 news/12225.bugfix.rst delete mode 100644 news/12252.trivial.rst delete mode 100644 news/12254.process.rst delete mode 100644 news/12261.trivial.rst delete mode 100644 news/12280.bugfix.rst delete mode 100644 news/12306.bugfix.rst delete mode 100644 news/12334.doc.rst delete mode 100644 news/12350.bugfix.rst delete mode 100644 news/12AE57EC-683C-4A8E-BCCB-851FCD0730B4.trivial.rst delete mode 100644 news/1F54AB69-21F3-49F6-B938-AB16E326F82C.trivial.rst delete mode 100644 news/2984.bugfix.rst delete mode 100644 news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst delete mode 100644 news/732404DE-8011-4146-8CAD-85D7756D88A6.trivial.rst delete mode 100644 news/80291DF4-7B0F-4268-B682-E1FCA1C3ACED.trivial.rst delete mode 100644 news/85F7E260-68FF-4C1E-A2CB-CF8634829D2D.trivial.rst delete mode 100644 news/E2B261CA-A0CF-4309-B808-1210C0B54632.trivial.rst delete mode 100644 news/certifi.vendor.rst delete mode 100644 news/d7179b28-bc23-46aa-9175-834117a42dbd.trivial.rst delete mode 100644 news/truststore.vendor.rst delete mode 100644 news/urllib3.vendor.rst delete mode 100644 news/zhsdgdlsjgksdfj.trivial.rst diff --git a/NEWS.rst b/NEWS.rst index fc3bb6697..27ac69d79 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,63 @@ .. towncrier release notes start +23.3 (2023-10-15) +================= + +Process +------- + +- Added reference to `vulnerability reporting guidelines `_ to pip's security policy. + +Deprecations and Removals +------------------------- + +- Drop a fallback to using SecureTransport on macOS. It was useful when pip detected OpenSSL older than 1.0.1, but the current pip does not support any Python version supporting such old OpenSSL versions. (`#12175 `_) + +Features +-------- + +- Improve extras resolution for multiple constraints on same base package. (`#11924 `_) +- Improve use of datastructures to make candidate selection 1.6x faster (`#12204 `_) +- Allow ``pip install --dry-run`` to use platform and ABI overriding options similar to ``--target``. (`#12215 `_) +- Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to PEP 592. (`#12224 `_) + +Bug Fixes +--------- + +- Ignore errors in temporary directory cleanup (show a warning instead). (`#11394 `_) +- Normalize extras according to :pep:`685` from package metadata in the resolver + for comparison. This ensures extras are correctly compared and merged as long + as the package providing the extra(s) is built with values normalized according + to the standard. Note, however, that this *does not* solve cases where the + package itself contains unnormalized extra values in the metadata. (`#11649 `_) +- Prevent downloading sdists twice when PEP 658 metadata is present. (`#11847 `_) +- Include all requested extras in the install report (``--report``). (`#11924 `_) +- Removed uses of ``datetime.datetime.utcnow`` from non-vendored code. (`#12005 `_) +- Consistently report whether a dependency comes from an extra. (`#12095 `_) +- Fix completion script for zsh (`#12166 `_) +- Fix improper handling of the new onexc argument of ``shutil.rmtree()`` in Python 3.12. (`#12187 `_) +- Filter out yanked links from the available versions error message: "(from versions: 1.0, 2.0, 3.0)" will not contain yanked versions conform PEP 592. The yanked versions (if any) will be mentioned in a separate error message. (`#12225 `_) +- Fix crash when the git version number contains something else than digits and dots. (`#12280 `_) +- Use ``-r=...`` instead of ``-r ...`` to specify references with Mercurial. (`#12306 `_) +- Redact password from URLs in some additional places. (`#12350 `_) +- pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). (`#2984 `_) + +Vendored Libraries +------------------ + +- Upgrade certifi to 2023.7.22 +- Add truststore 0.8.0 +- Upgrade urllib3 to 1.26.17 + +Improved Documentation +---------------------- + +- Document that ``pip search`` support has been removed from PyPI (`#12059 `_) +- Clarify --prefer-binary in CLI and docs (`#12122 `_) +- Document that using OS-provided Python can cause pip's test suite to report false failures. (`#12334 `_) + + 23.2.1 (2023-07-22) =================== diff --git a/news/11394.bugfix.rst b/news/11394.bugfix.rst deleted file mode 100644 index 9f2501db4..000000000 --- a/news/11394.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore errors in temporary directory cleanup (show a warning instead). diff --git a/news/11649.bugfix.rst b/news/11649.bugfix.rst deleted file mode 100644 index 65511711f..000000000 --- a/news/11649.bugfix.rst +++ /dev/null @@ -1,5 +0,0 @@ -Normalize extras according to :pep:`685` from package metadata in the resolver -for comparison. This ensures extras are correctly compared and merged as long -as the package providing the extra(s) is built with values normalized according -to the standard. Note, however, that this *does not* solve cases where the -package itself contains unnormalized extra values in the metadata. diff --git a/news/11847.bugfix.rst b/news/11847.bugfix.rst deleted file mode 100644 index 1f384835f..000000000 --- a/news/11847.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent downloading sdists twice when PEP 658 metadata is present. diff --git a/news/11924.bugfix.rst b/news/11924.bugfix.rst deleted file mode 100644 index 7a9ee3151..000000000 --- a/news/11924.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Include all requested extras in the install report (``--report``). diff --git a/news/11924.feature.rst b/news/11924.feature.rst deleted file mode 100644 index 30bc60e6b..000000000 --- a/news/11924.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Improve extras resolution for multiple constraints on same base package. diff --git a/news/12005.bugfix.rst b/news/12005.bugfix.rst deleted file mode 100644 index 98a3e5112..000000000 --- a/news/12005.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Removed uses of ``datetime.datetime.utcnow`` from non-vendored code. diff --git a/news/12059.doc.rst b/news/12059.doc.rst deleted file mode 100644 index bf3a8d3e6..000000000 --- a/news/12059.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Document that ``pip search`` support has been removed from PyPI diff --git a/news/12095.bugfix.rst b/news/12095.bugfix.rst deleted file mode 100644 index 1f5018326..000000000 --- a/news/12095.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Consistently report whether a dependency comes from an extra. diff --git a/news/12122.doc.rst b/news/12122.doc.rst deleted file mode 100644 index 49a3308a2..000000000 --- a/news/12122.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Clarify --prefer-binary in CLI and docs diff --git a/news/12155.trivial.rst b/news/12155.trivial.rst deleted file mode 100644 index 5f77231c8..000000000 --- a/news/12155.trivial.rst +++ /dev/null @@ -1,6 +0,0 @@ -The metadata-fetching log message is moved to the VERBOSE level and now hidden -by default. The more significant information in this message to most users are -already available in surrounding logs (the package name and version of the -metadata being fetched), while the URL to the exact metadata file is generally -too long and clutters the output. The message can be brought back with -``--verbose``. diff --git a/news/12166.bugfix.rst b/news/12166.bugfix.rst deleted file mode 100644 index 491597c7f..000000000 --- a/news/12166.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix completion script for zsh diff --git a/news/12175.removal.rst b/news/12175.removal.rst deleted file mode 100644 index bf3500f35..000000000 --- a/news/12175.removal.rst +++ /dev/null @@ -1 +0,0 @@ -Drop a fallback to using SecureTransport on macOS. It was useful when pip detected OpenSSL older than 1.0.1, but the current pip does not support any Python version supporting such old OpenSSL versions. diff --git a/news/12183.trivial.rst b/news/12183.trivial.rst deleted file mode 100644 index c22e854c9..000000000 --- a/news/12183.trivial.rst +++ /dev/null @@ -1 +0,0 @@ -Add test cases for some behaviors of ``install --dry-run`` and ``--use-feature=fast-deps``. diff --git a/news/12187.bugfix.rst b/news/12187.bugfix.rst deleted file mode 100644 index b4d106b97..000000000 --- a/news/12187.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix improper handling of the new onexc argument of ``shutil.rmtree()`` in Python 3.12. diff --git a/news/12194.trivial.rst b/news/12194.trivial.rst deleted file mode 100644 index dfe5bbf1f..000000000 --- a/news/12194.trivial.rst +++ /dev/null @@ -1 +0,0 @@ -Add lots of comments to the ``BuildTracker``. diff --git a/news/12204.feature.rst b/news/12204.feature.rst deleted file mode 100644 index 6ffdf5123..000000000 --- a/news/12204.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Improve use of datastructures to make candidate selection 1.6x faster diff --git a/news/12215.feature.rst b/news/12215.feature.rst deleted file mode 100644 index 407dc903e..000000000 --- a/news/12215.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Allow ``pip install --dry-run`` to use platform and ABI overriding options similar to ``--target``. diff --git a/news/12224.feature.rst b/news/12224.feature.rst deleted file mode 100644 index d87426578..000000000 --- a/news/12224.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to PEP 592. diff --git a/news/12225.bugfix.rst b/news/12225.bugfix.rst deleted file mode 100644 index e1e0c323d..000000000 --- a/news/12225.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Filter out yanked links from the available versions error message: "(from versions: 1.0, 2.0, 3.0)" will not contain yanked versions conform PEP 592. The yanked versions (if any) will be mentioned in a separate error message. diff --git a/news/12252.trivial.rst b/news/12252.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/12254.process.rst b/news/12254.process.rst deleted file mode 100644 index e54690268..000000000 --- a/news/12254.process.rst +++ /dev/null @@ -1 +0,0 @@ -Added reference to `vulnerability reporting guidelines `_ to pip's security policy. diff --git a/news/12261.trivial.rst b/news/12261.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/12280.bugfix.rst b/news/12280.bugfix.rst deleted file mode 100644 index 77de283d3..000000000 --- a/news/12280.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash when the git version number contains something else than digits and dots. diff --git a/news/12306.bugfix.rst b/news/12306.bugfix.rst deleted file mode 100644 index eb6eecaaf..000000000 --- a/news/12306.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Use ``-r=...`` instead of ``-r ...`` to specify references with Mercurial. diff --git a/news/12334.doc.rst b/news/12334.doc.rst deleted file mode 100644 index ff3d877e5..000000000 --- a/news/12334.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Document that using OS-provided Python can cause pip's test suite to report false failures. diff --git a/news/12350.bugfix.rst b/news/12350.bugfix.rst deleted file mode 100644 index 3fb16b4ed..000000000 --- a/news/12350.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Redact password from URLs in some additional places. diff --git a/news/12AE57EC-683C-4A8E-BCCB-851FCD0730B4.trivial.rst b/news/12AE57EC-683C-4A8E-BCCB-851FCD0730B4.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/1F54AB69-21F3-49F6-B938-AB16E326F82C.trivial.rst b/news/1F54AB69-21F3-49F6-B938-AB16E326F82C.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/2984.bugfix.rst b/news/2984.bugfix.rst deleted file mode 100644 index cce561815..000000000 --- a/news/2984.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). diff --git a/news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst b/news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst deleted file mode 100644 index 7f6c1d561..000000000 --- a/news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst +++ /dev/null @@ -1 +0,0 @@ -Add ruff rules ASYNC,C4,C90,PERF,PLE,PLR for minor optimizations and to set upper limits on code complexity. diff --git a/news/732404DE-8011-4146-8CAD-85D7756D88A6.trivial.rst b/news/732404DE-8011-4146-8CAD-85D7756D88A6.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/80291DF4-7B0F-4268-B682-E1FCA1C3ACED.trivial.rst b/news/80291DF4-7B0F-4268-B682-E1FCA1C3ACED.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/85F7E260-68FF-4C1E-A2CB-CF8634829D2D.trivial.rst b/news/85F7E260-68FF-4C1E-A2CB-CF8634829D2D.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/E2B261CA-A0CF-4309-B808-1210C0B54632.trivial.rst b/news/E2B261CA-A0CF-4309-B808-1210C0B54632.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/certifi.vendor.rst b/news/certifi.vendor.rst deleted file mode 100644 index aacd17183..000000000 --- a/news/certifi.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade certifi to 2023.7.22 diff --git a/news/d7179b28-bc23-46aa-9175-834117a42dbd.trivial.rst b/news/d7179b28-bc23-46aa-9175-834117a42dbd.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/truststore.vendor.rst b/news/truststore.vendor.rst deleted file mode 100644 index 63c71d72d..000000000 --- a/news/truststore.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Add truststore 0.8.0 diff --git a/news/urllib3.vendor.rst b/news/urllib3.vendor.rst deleted file mode 100644 index 37032f67a..000000000 --- a/news/urllib3.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade urllib3 to 1.26.17 diff --git a/news/zhsdgdlsjgksdfj.trivial.rst b/news/zhsdgdlsjgksdfj.trivial.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 00ce8ad45..62498a779 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "23.3.dev0" +__version__ = "23.3" def main(args: Optional[List[str]] = None) -> int: From c0cce3ca6048b27d80b78a88d6af1b25b10a2a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 10:23:09 +0200 Subject: [PATCH 144/156] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 62498a779..46e560149 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "23.3" +__version__ = "24.0.dev0" def main(args: Optional[List[str]] = None) -> int: From 9b0abc8c40459dd16a9c1205e15f6d3363bf202e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 18:44:34 +0200 Subject: [PATCH 145/156] Build using `build` Update the build-release nox session to build using `build` instead of a direct setup.py call. --- noxfile.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index a3e7ceab4..878dbbd0a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -322,7 +322,7 @@ def build_release(session: nox.Session) -> None: ) session.log("# Install dependencies") - session.install("setuptools", "wheel", "twine") + session.install("build", "twine") with release.isolated_temporary_checkout(session, version) as build_dir: session.log( @@ -358,8 +358,7 @@ def build_dists(session: nox.Session) -> List[str]: ) session.log("# Build distributions") - session.install("setuptools", "wheel") - session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True) + session.run("python", "-m", "build", silent=True) produced_dists = glob.glob("dist/*") session.log(f"# Verify distributions: {', '.join(produced_dists)}") From e1e227d7d6b5ae04ae3a2104bf8185622201f5f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 18:48:53 +0200 Subject: [PATCH 146/156] Clarify changelog --- NEWS.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 27ac69d79..5004bd9f7 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -26,8 +26,8 @@ Features -------- - Improve extras resolution for multiple constraints on same base package. (`#11924 `_) -- Improve use of datastructures to make candidate selection 1.6x faster (`#12204 `_) -- Allow ``pip install --dry-run`` to use platform and ABI overriding options similar to ``--target``. (`#12215 `_) +- Improve use of datastructures to make candidate selection 1.6x faster. (`#12204 `_) +- Allow ``pip install --dry-run`` to use platform and ABI overriding options. (`#12215 `_) - Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to PEP 592. (`#12224 `_) Bug Fixes From a982c7bc3550afb27a3a792d84fe91bf7c3254ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 15 Oct 2023 19:17:15 +0200 Subject: [PATCH 147/156] Add a few PEP links in the changelog --- NEWS.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 5004bd9f7..4f496273c 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -28,7 +28,7 @@ Features - Improve extras resolution for multiple constraints on same base package. (`#11924 `_) - Improve use of datastructures to make candidate selection 1.6x faster. (`#12204 `_) - Allow ``pip install --dry-run`` to use platform and ABI overriding options. (`#12215 `_) -- Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to PEP 592. (`#12224 `_) +- Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to :pep:`592`. (`#12224 `_) Bug Fixes --------- @@ -39,7 +39,7 @@ Bug Fixes as the package providing the extra(s) is built with values normalized according to the standard. Note, however, that this *does not* solve cases where the package itself contains unnormalized extra values in the metadata. (`#11649 `_) -- Prevent downloading sdists twice when PEP 658 metadata is present. (`#11847 `_) +- Prevent downloading sdists twice when :pep:`658` metadata is present. (`#11847 `_) - Include all requested extras in the install report (``--report``). (`#11924 `_) - Removed uses of ``datetime.datetime.utcnow`` from non-vendored code. (`#12005 `_) - Consistently report whether a dependency comes from an extra. (`#12095 `_) @@ -72,7 +72,7 @@ Improved Documentation Bug Fixes --------- -- Disable PEP 658 metadata fetching with the legacy resolver. (`#12156 `_) +- Disable :pep:`658` metadata fetching with the legacy resolver. (`#12156 `_) 23.2 (2023-07-15) @@ -102,11 +102,11 @@ Bug Fixes --------- - Fix ``pip completion --zsh``. (`#11417 `_) -- Prevent downloading files twice when PEP 658 metadata is present (`#11847 `_) +- Prevent downloading files twice when :pep:`658` metadata is present (`#11847 `_) - Add permission check before configuration (`#11920 `_) - Fix deprecation warnings in Python 3.12 for usage of shutil.rmtree (`#11957 `_) - Ignore invalid or unreadable ``origin.json`` files in the cache of locally built wheels. (`#11985 `_) -- Fix installation of packages with PEP658 metadata using non-canonicalized names (`#12038 `_) +- Fix installation of packages with :pep:`658` metadata using non-canonicalized names (`#12038 `_) - Correctly parse ``dist-info-metadata`` values from JSON-format index data. (`#12042 `_) - Fail with an error if the ``--python`` option is specified after the subcommand name. (`#12067 `_) - Fix slowness when using ``importlib.metadata`` (the default way for pip to read metadata in Python 3.11+) and there is a large overlap between already installed and to-be-installed packages. (`#12079 `_) @@ -277,7 +277,7 @@ Features - Change the hashes in the installation report to be a mapping. Emit the ``archive_info.hashes`` dictionary in ``direct_url.json``. (`#11312 `_) -- Implement logic to read the ``EXTERNALLY-MANAGED`` file as specified in PEP 668. +- Implement logic to read the ``EXTERNALLY-MANAGED`` file as specified in :pep:`668`. This allows a downstream Python distributor to prevent users from using pip to modify the externally managed environment. (`#11381 `_) - Enable the use of ``keyring`` found on ``PATH``. This allows ``keyring`` @@ -293,7 +293,7 @@ Bug Fixes - Use the "venv" scheme if available to obtain prefixed lib paths. (`#11598 `_) - Deprecated a historical ambiguity in how ``egg`` fragments in URL-style requirements are formatted and handled. ``egg`` fragments that do not look - like PEP 508 names now produce a deprecation warning. (`#11617 `_) + like :pep:`508` names now produce a deprecation warning. (`#11617 `_) - Fix scripts path in isolated build environment on Debian. (`#11623 `_) - Make ``pip show`` show the editable location if package is editable (`#11638 `_) - Stop checking that ``wheel`` is present when ``build-system.requires`` From fb06d12d5a32581ae531fc26143c14ac6c8ea8fe Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 17 Oct 2023 11:04:34 +0100 Subject: [PATCH 148/156] Handle ISO formats with a trailing Z --- news/12338.bugfix.rst | 1 + src/pip/_internal/self_outdated_check.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 news/12338.bugfix.rst diff --git a/news/12338.bugfix.rst b/news/12338.bugfix.rst new file mode 100644 index 000000000..cd9a8c10b --- /dev/null +++ b/news/12338.bugfix.rst @@ -0,0 +1 @@ +Handle a timezone indicator of Z when parsing dates in the self check. diff --git a/src/pip/_internal/self_outdated_check.py b/src/pip/_internal/self_outdated_check.py index cb18edbed..0f64ae0e6 100644 --- a/src/pip/_internal/self_outdated_check.py +++ b/src/pip/_internal/self_outdated_check.py @@ -39,6 +39,15 @@ def _get_statefile_name(key: str) -> str: return name +def _convert_date(isodate: str) -> datetime.datetime: + """Convert an ISO format string to a date. + + Handles the format 2020-01-22T14:24:01Z (trailing Z) + which is not supported by older versions of fromisoformat. + """ + return datetime.datetime.fromisoformat(isodate.replace("Z", "+00:00")) + + class SelfCheckState: def __init__(self, cache_dir: str) -> None: self._state: Dict[str, Any] = {} @@ -73,7 +82,7 @@ class SelfCheckState: return None # Determine if we need to refresh the state - last_check = datetime.datetime.fromisoformat(self._state["last_check"]) + last_check = _convert_date(self._state["last_check"]) time_since_last_check = current_time - last_check if time_since_last_check > _WEEK: return None From 5e7cc16c3b4442055a4a9892e9231758b6714e28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 Oct 2023 18:14:22 -0400 Subject: [PATCH 149/156] Fix parallel pip cache downloads causing crash (#12364) Co-authored-by: Itamar Turner-Trauring --- news/12361.bugfix.rst | 1 + src/pip/_internal/network/cache.py | 28 ++++++++++++++++++++++++---- tests/unit/test_network_cache.py | 11 +++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 news/12361.bugfix.rst diff --git a/news/12361.bugfix.rst b/news/12361.bugfix.rst new file mode 100644 index 000000000..59575e80b --- /dev/null +++ b/news/12361.bugfix.rst @@ -0,0 +1 @@ +Fix bug where installing the same package at the same time with multiple pip processes could fail. diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index a4d136205..4d0fb545d 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -33,6 +33,18 @@ class SafeFileCache(SeparateBodyBaseCache): """ A file based cache which is safe to use even when the target directory may not be accessible or writable. + + There is a race condition when two processes try to write and/or read the + same entry at the same time, since each entry consists of two separate + files (https://github.com/psf/cachecontrol/issues/324). We therefore have + additional logic that makes sure that both files to be present before + returning an entry; this fixes the read side of the race condition. + + For the write side, we assume that the server will only ever return the + same data for the same URL, which ought to be the case for files pip is + downloading. PyPI does not have a mechanism to swap out a wheel for + another wheel, for example. If this assumption is not true, the + CacheControl issue will need to be fixed. """ def __init__(self, directory: str) -> None: @@ -49,9 +61,13 @@ class SafeFileCache(SeparateBodyBaseCache): return os.path.join(self.directory, *parts) def get(self, key: str) -> Optional[bytes]: - path = self._get_cache_path(key) + # The cache entry is only valid if both metadata and body exist. + metadata_path = self._get_cache_path(key) + body_path = metadata_path + ".body" + if not (os.path.exists(metadata_path) and os.path.exists(body_path)): + return None with suppressed_cache_errors(): - with open(path, "rb") as f: + with open(metadata_path, "rb") as f: return f.read() def _write(self, path: str, data: bytes) -> None: @@ -77,9 +93,13 @@ class SafeFileCache(SeparateBodyBaseCache): os.remove(path + ".body") def get_body(self, key: str) -> Optional[BinaryIO]: - path = self._get_cache_path(key) + ".body" + # The cache entry is only valid if both metadata and body exist. + metadata_path = self._get_cache_path(key) + body_path = metadata_path + ".body" + if not (os.path.exists(metadata_path) and os.path.exists(body_path)): + return None with suppressed_cache_errors(): - return open(path, "rb") + return open(body_path, "rb") def set_body(self, key: str, body: bytes) -> None: path = self._get_cache_path(key) + ".body" diff --git a/tests/unit/test_network_cache.py b/tests/unit/test_network_cache.py index aa849f3b0..6a816b300 100644 --- a/tests/unit/test_network_cache.py +++ b/tests/unit/test_network_cache.py @@ -27,6 +27,11 @@ class TestSafeFileCache: cache = SafeFileCache(os.fspath(cache_tmpdir)) assert cache.get("test key") is None cache.set("test key", b"a test string") + # Body hasn't been stored yet, so the entry isn't valid yet + assert cache.get("test key") is None + + # With a body, the cache entry is valid: + cache.set_body("test key", b"body") assert cache.get("test key") == b"a test string" cache.delete("test key") assert cache.get("test key") is None @@ -35,6 +40,12 @@ class TestSafeFileCache: cache = SafeFileCache(os.fspath(cache_tmpdir)) assert cache.get_body("test key") is None cache.set_body("test key", b"a test string") + # Metadata isn't available, so the entry isn't valid yet (this + # shouldn't happen, but just in case) + assert cache.get_body("test key") is None + + # With metadata, the cache entry is valid: + cache.set("test key", b"metadata") body = cache.get_body("test key") assert body is not None with body: From 5364f26f9631dc07ed1bdfc88e1bec1bead2bce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 21 Oct 2023 12:57:31 +0200 Subject: [PATCH 150/156] Bump for release --- NEWS.rst | 10 ++++++++++ news/12338.bugfix.rst | 1 - news/12361.bugfix.rst | 1 - src/pip/__init__.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 news/12338.bugfix.rst delete mode 100644 news/12361.bugfix.rst diff --git a/NEWS.rst b/NEWS.rst index 4f496273c..26d00a3f1 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,16 @@ .. towncrier release notes start +23.3.1 (2023-10-21) +=================== + +Bug Fixes +--------- + +- Handle a timezone indicator of Z when parsing dates in the self check. (`#12338 `_) +- Fix bug where installing the same package at the same time with multiple pip processes could fail. (`#12361 `_) + + 23.3 (2023-10-15) ================= diff --git a/news/12338.bugfix.rst b/news/12338.bugfix.rst deleted file mode 100644 index cd9a8c10b..000000000 --- a/news/12338.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Handle a timezone indicator of Z when parsing dates in the self check. diff --git a/news/12361.bugfix.rst b/news/12361.bugfix.rst deleted file mode 100644 index 59575e80b..000000000 --- a/news/12361.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug where installing the same package at the same time with multiple pip processes could fail. diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 46e560149..f1263cdc1 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "24.0.dev0" +__version__ = "23.3.1" def main(args: Optional[List[str]] = None) -> int: From 576dbd813c3a0e989c14a36f6d214ad66309ff97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 21 Oct 2023 12:57:41 +0200 Subject: [PATCH 151/156] Bump for development --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index f1263cdc1..46e560149 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "23.3.1" +__version__ = "24.0.dev0" def main(args: Optional[List[str]] = None) -> int: From 6dbd9c68f085c5bf304247bf7c7933842092efb2 Mon Sep 17 00:00:00 2001 From: efflamlemaillet <6533295+efflamlemaillet@users.noreply.github.com> Date: Fri, 27 Oct 2023 11:08:17 +0200 Subject: [PATCH 152/156] Fix hg: "parse error at 0: not a prefix:" (#12373) Use two hypen argument `--rev=` instead of `-r=` Co-authored-by: Efflam Lemaillet Co-authored-by: Pradyun Gedam --- news/370392cf-52cd-402c-b402-06d2ff398f89.bugfix.rst | 1 + src/pip/_internal/vcs/mercurial.py | 2 +- tests/unit/test_vcs.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 news/370392cf-52cd-402c-b402-06d2ff398f89.bugfix.rst diff --git a/news/370392cf-52cd-402c-b402-06d2ff398f89.bugfix.rst b/news/370392cf-52cd-402c-b402-06d2ff398f89.bugfix.rst new file mode 100644 index 000000000..76a8e6b96 --- /dev/null +++ b/news/370392cf-52cd-402c-b402-06d2ff398f89.bugfix.rst @@ -0,0 +1 @@ +Fix mercurial revision "parse error": use ``--rev={ref}`` instead of ``-r={ref}`` diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index e440c1221..c183d41d0 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -31,7 +31,7 @@ class Mercurial(VersionControl): @staticmethod def get_base_rev_args(rev: str) -> List[str]: - return [f"-r={rev}"] + return [f"--rev={rev}"] def fetch_new( self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 4a3750f2d..5291f129c 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -66,7 +66,7 @@ def test_rev_options_repr() -> None: # First check VCS-specific RevOptions behavior. (Bazaar, [], ["-r", "123"], {}), (Git, ["HEAD"], ["123"], {}), - (Mercurial, [], ["-r=123"], {}), + (Mercurial, [], ["--rev=123"], {}), (Subversion, [], ["-r", "123"], {}), # Test extra_args. For this, test using a single VersionControl class. ( From fd77ebfc742de4d76ff976de22e86d116e0faad3 Mon Sep 17 00:00:00 2001 From: Dale <70705126+dalebrydon@users.noreply.github.com> Date: Fri, 27 Oct 2023 08:59:56 -0400 Subject: [PATCH 153/156] Rework the functionality of PIP_CONFIG_FILE (#11850) --- docs/html/topics/configuration.md | 19 ++++++++++++------- news/11815.doc.rst | 1 + src/pip/_internal/configuration.py | 26 ++++++++++++++------------ 3 files changed, 27 insertions(+), 19 deletions(-) create mode 100644 news/11815.doc.rst diff --git a/docs/html/topics/configuration.md b/docs/html/topics/configuration.md index e4aafcd2b..8b54db56c 100644 --- a/docs/html/topics/configuration.md +++ b/docs/html/topics/configuration.md @@ -19,8 +19,8 @@ and how they are related to pip's various command line options. ## Configuration Files -Configuration files can change the default values for command line option. -They are written using a standard INI style configuration files. +Configuration files can change the default values for command line options. +The files are written using standard INI format. pip has 3 "levels" of configuration files: @@ -28,11 +28,15 @@ pip has 3 "levels" of configuration files: - `user`: per-user configuration file. - `site`: per-environment configuration file; i.e. per-virtualenv. +Additionally, environment variables can be specified which will override any of the above. + ### Location pip's configuration files are located in fairly standard locations. This location is different on different operating systems, and has some additional -complexity for backwards compatibility reasons. +complexity for backwards compatibility reasons. Note that if user config files +exist in both the legacy and current locations, values in the current file +will override values in the legacy file. ```{tab} Unix @@ -88,9 +92,10 @@ Site ### `PIP_CONFIG_FILE` Additionally, the environment variable `PIP_CONFIG_FILE` can be used to specify -a configuration file that's loaded first, and whose values are overridden by -the values set in the aforementioned files. Setting this to {any}`os.devnull` -disables the loading of _all_ configuration files. +a configuration file that's loaded last, and whose values override the values +set in the aforementioned files. Setting this to {any}`os.devnull` +disables the loading of _all_ configuration files. Note that if a file exists +at the location that this is set to, the user config file will not be loaded. (config-precedence)= @@ -99,10 +104,10 @@ disables the loading of _all_ configuration files. When multiple configuration files are found, pip combines them in the following order: -- `PIP_CONFIG_FILE`, if given. - Global - User - Site +- `PIP_CONFIG_FILE`, if given. Each file read overrides any values read from previous files, so if the global timeout is specified in both the global file and the per-user file diff --git a/news/11815.doc.rst b/news/11815.doc.rst new file mode 100644 index 000000000..8e7e8d21b --- /dev/null +++ b/news/11815.doc.rst @@ -0,0 +1 @@ +Fix explanation of how PIP_CONFIG_FILE works diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 96f824955..124a7ca5d 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -327,33 +327,35 @@ class Configuration: def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]: """Yields variant and configuration files associated with it. - This should be treated like items of a dictionary. + This should be treated like items of a dictionary. The order + here doesn't affect what gets overridden. That is controlled + by OVERRIDE_ORDER. However this does control the order they are + displayed to the user. It's probably most ergononmic to display + things in the same order as OVERRIDE_ORDER """ # SMELL: Move the conditions out of this function - # environment variables have the lowest priority - config_file = os.environ.get("PIP_CONFIG_FILE", None) - if config_file is not None: - yield kinds.ENV, [config_file] - else: - yield kinds.ENV, [] - + env_config_file = os.environ.get("PIP_CONFIG_FILE", None) config_files = get_configuration_files() - # at the base we have any global configuration yield kinds.GLOBAL, config_files[kinds.GLOBAL] - # per-user configuration next + # per-user config is not loaded when env_config_file exists should_load_user_config = not self.isolated and not ( - config_file and os.path.exists(config_file) + env_config_file and os.path.exists(env_config_file) ) if should_load_user_config: # The legacy config file is overridden by the new config file yield kinds.USER, config_files[kinds.USER] - # finally virtualenv configuration first trumping others + # virtualenv config yield kinds.SITE, config_files[kinds.SITE] + if env_config_file is not None: + yield kinds.ENV, [env_config_file] + else: + yield kinds.ENV, [] + def get_values_in_config(self, variant: Kind) -> Dict[str, Any]: """Get values present in a config file""" return self._config[variant] From 9685f64fe8c78e1e39cd9b32e5615f42e0a01f1c Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Mon, 6 Nov 2023 04:30:05 -0500 Subject: [PATCH 154/156] Update ruff and config (#12390) --- .pre-commit-config.yaml | 3 ++- news/12390.trivial.rst | 1 + pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 news/12390.trivial.rst diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c576d90a..999bd8b1d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,9 +22,10 @@ repos: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.1.4 hooks: - id: ruff + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.961 diff --git a/news/12390.trivial.rst b/news/12390.trivial.rst new file mode 100644 index 000000000..52b21413c --- /dev/null +++ b/news/12390.trivial.rst @@ -0,0 +1 @@ +Update ruff versions and config for dev diff --git a/pyproject.toml b/pyproject.toml index b720c4602..d22e5c668 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,8 +84,8 @@ ignore = [ "B020", "B904", # Ruff enables opinionated warnings by default "B905", # Ruff enables opinionated warnings by default - "G202", ] +target-version = "py37" line-length = 88 select = [ "ASYNC", From 68529081c27e2372971f114d5464c0850837405b Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Tue, 7 Nov 2023 04:14:56 -0500 Subject: [PATCH 155/156] Enforce f-strings via Ruff (#12393) --- docs/pip_sphinxext.py | 11 +--- news/12393.trivial.rst | 1 + pyproject.toml | 1 + setup.py | 2 +- src/pip/_internal/cli/cmdoptions.py | 9 +-- src/pip/_internal/cli/parser.py | 8 +-- src/pip/_internal/commands/cache.py | 2 +- src/pip/_internal/commands/configuration.py | 10 ++- src/pip/_internal/commands/debug.py | 8 +-- src/pip/_internal/commands/index.py | 4 +- src/pip/_internal/commands/install.py | 6 +- src/pip/_internal/configuration.py | 4 +- src/pip/_internal/exceptions.py | 13 ++-- src/pip/_internal/index/package_finder.py | 8 +-- src/pip/_internal/models/candidate.py | 6 +- src/pip/_internal/models/direct_url.py | 4 +- src/pip/_internal/models/format_control.py | 4 +- src/pip/_internal/models/link.py | 4 +- src/pip/_internal/network/download.py | 2 +- src/pip/_internal/operations/install/wheel.py | 20 +++--- src/pip/_internal/operations/prepare.py | 10 +-- src/pip/_internal/req/constructors.py | 2 +- src/pip/_internal/req/req_install.py | 8 +-- src/pip/_internal/req/req_uninstall.py | 8 +-- .../_internal/resolution/legacy/resolver.py | 14 ++--- .../resolution/resolvelib/candidates.py | 16 +---- .../resolution/resolvelib/factory.py | 4 +- .../resolution/resolvelib/requirements.py | 20 ++---- src/pip/_internal/utils/misc.py | 20 +++--- src/pip/_internal/utils/wheel.py | 6 +- src/pip/_internal/vcs/versioncontrol.py | 10 +-- src/pip/_internal/wheel_builder.py | 11 ++-- tests/conftest.py | 2 +- tests/functional/test_cli.py | 8 +-- tests/functional/test_completion.py | 2 +- tests/functional/test_debug.py | 2 +- tests/functional/test_freeze.py | 28 ++++----- tests/functional/test_install.py | 12 ++-- tests/functional/test_install_config.py | 18 +++--- tests/functional/test_install_index.py | 8 +-- tests/functional/test_install_reqs.py | 8 +-- tests/functional/test_install_wheel.py | 4 +- tests/functional/test_new_resolver.py | 2 +- tests/functional/test_new_resolver_errors.py | 6 +- tests/functional/test_new_resolver_hashes.py | 63 ++++++------------- tests/functional/test_new_resolver_target.py | 7 +-- tests/functional/test_pep517.py | 12 ++-- tests/functional/test_uninstall.py | 4 +- tests/functional/test_wheel.py | 4 +- tests/lib/__init__.py | 30 ++++----- tests/lib/local_repos.py | 2 +- tests/lib/server.py | 2 +- tests/lib/test_lib.py | 4 +- tests/lib/wheel.py | 2 +- tests/unit/test_collector.py | 12 ++-- tests/unit/test_link.py | 5 +- tests/unit/test_network_utils.py | 4 +- tests/unit/test_req.py | 4 +- tests/unit/test_req_file.py | 10 ++- tests/unit/test_resolution_legacy_resolver.py | 4 +- tests/unit/test_wheel.py | 8 +-- tools/release/check_version.py | 2 +- 62 files changed, 201 insertions(+), 334 deletions(-) create mode 100644 news/12393.trivial.rst diff --git a/docs/pip_sphinxext.py b/docs/pip_sphinxext.py index 2e5597022..fe3f41e8b 100644 --- a/docs/pip_sphinxext.py +++ b/docs/pip_sphinxext.py @@ -194,22 +194,17 @@ class PipReqFileOptionsReference(PipOptions): opt = option() opt_name = opt._long_opts[0] if opt._short_opts: - short_opt_name = "{}, ".format(opt._short_opts[0]) + short_opt_name = f"{opt._short_opts[0]}, " else: short_opt_name = "" if option in cmdoptions.general_group["options"]: prefix = "" else: - prefix = "{}_".format(self.determine_opt_prefix(opt_name)) + prefix = f"{self.determine_opt_prefix(opt_name)}_" self.view_list.append( - "* :ref:`{short}{long}<{prefix}{opt_name}>`".format( - short=short_opt_name, - long=opt_name, - prefix=prefix, - opt_name=opt_name, - ), + f"* :ref:`{short_opt_name}{opt_name}<{prefix}{opt_name}>`", "\n", ) diff --git a/news/12393.trivial.rst b/news/12393.trivial.rst new file mode 100644 index 000000000..15452737a --- /dev/null +++ b/news/12393.trivial.rst @@ -0,0 +1 @@ +Enforce and update code to use f-strings via Ruff rule UP032 diff --git a/pyproject.toml b/pyproject.toml index d22e5c668..0ac70174a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,7 @@ select = [ "PLR0", "W", "RUF100", + "UP032", ] [tool.ruff.isort] diff --git a/setup.py b/setup.py index d73c77b73..569389b1e 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ setup( entry_points={ "console_scripts": [ "pip=pip._internal.cli.main:main", - "pip{}=pip._internal.cli.main:main".format(sys.version_info[0]), + f"pip{sys.version_info[0]}=pip._internal.cli.main:main", "pip{}.{}=pip._internal.cli.main:main".format(*sys.version_info[:2]), ], }, diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 8fb16dc4a..d05e502f9 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -582,10 +582,7 @@ def _handle_python_version( """ version_info, error_msg = _convert_python_version(value) if error_msg is not None: - msg = "invalid --python-version value: {!r}: {}".format( - value, - error_msg, - ) + msg = f"invalid --python-version value: {value!r}: {error_msg}" raise_option_error(parser, option=option, msg=msg) parser.values.python_version = version_info @@ -921,9 +918,9 @@ def _handle_merge_hash( algo, digest = value.split(":", 1) except ValueError: parser.error( - "Arguments to {} must be a hash name " + f"Arguments to {opt_str} must be a hash name " "followed by a value, like --hash=sha256:" - "abcde...".format(opt_str) + "abcde..." ) if algo not in STRONG_HASHES: parser.error( diff --git a/src/pip/_internal/cli/parser.py b/src/pip/_internal/cli/parser.py index 64cf97197..ae554b24c 100644 --- a/src/pip/_internal/cli/parser.py +++ b/src/pip/_internal/cli/parser.py @@ -229,9 +229,9 @@ class ConfigOptionParser(CustomOptionParser): val = strtobool(val) except ValueError: self.error( - "{} is not a valid value for {} option, " + f"{val} is not a valid value for {key} option, " "please specify a boolean value like yes/no, " - "true/false or 1/0 instead.".format(val, key) + "true/false or 1/0 instead." ) elif option.action == "count": with suppress(ValueError): @@ -240,10 +240,10 @@ class ConfigOptionParser(CustomOptionParser): val = int(val) if not isinstance(val, int) or val < 0: self.error( - "{} is not a valid value for {} option, " + f"{val} is not a valid value for {key} option, " "please instead specify either a non-negative integer " "or a boolean value like yes/no or false/true " - "which is equivalent to 1/0.".format(val, key) + "which is equivalent to 1/0." ) elif option.action == "append": val = val.split() diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 1f3b5fe14..328336152 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -175,7 +175,7 @@ class CacheCommand(Command): files += self._find_http_files(options) else: # Add the pattern to the log message - no_matching_msg += ' for pattern "{}"'.format(args[0]) + no_matching_msg += f' for pattern "{args[0]}"' if not files: logger.warning(no_matching_msg) diff --git a/src/pip/_internal/commands/configuration.py b/src/pip/_internal/commands/configuration.py index 84b134e49..1a1dc6b6c 100644 --- a/src/pip/_internal/commands/configuration.py +++ b/src/pip/_internal/commands/configuration.py @@ -242,17 +242,15 @@ class ConfigurationCommand(Command): e.filename = editor raise except subprocess.CalledProcessError as e: - raise PipError( - "Editor Subprocess exited with exit code {}".format(e.returncode) - ) + raise PipError(f"Editor Subprocess exited with exit code {e.returncode}") def _get_n_args(self, args: List[str], example: str, n: int) -> Any: """Helper to make sure the command got the right number of arguments""" if len(args) != n: msg = ( - "Got unexpected number of arguments, expected {}. " - '(example: "{} config {}")' - ).format(n, get_prog(), example) + f"Got unexpected number of arguments, expected {n}. " + f'(example: "{get_prog()} config {example}")' + ) raise PipError(msg) if n == 1: diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index 5dc91bf49..7e5271c98 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -95,7 +95,7 @@ def show_actual_vendor_versions(vendor_txt_versions: Dict[str, str]) -> None: elif parse_version(actual_version) != parse_version(expected_version): extra_message = ( " (CONFLICT: vendor.txt suggests version should" - " be {})".format(expected_version) + f" be {expected_version})" ) logger.info("%s==%s%s", module_name, actual_version, extra_message) @@ -120,7 +120,7 @@ def show_tags(options: Values) -> None: if formatted_target: suffix = f" (target: {formatted_target})" - msg = "Compatible tags: {}{}".format(len(tags), suffix) + msg = f"Compatible tags: {len(tags)}{suffix}" logger.info(msg) if options.verbose < 1 and len(tags) > tag_limit: @@ -134,9 +134,7 @@ def show_tags(options: Values) -> None: logger.info(str(tag)) if tags_limited: - msg = ( - "...\n[First {tag_limit} tags shown. Pass --verbose to show all.]" - ).format(tag_limit=tag_limit) + msg = f"...\n[First {tag_limit} tags shown. Pass --verbose to show all.]" logger.info(msg) diff --git a/src/pip/_internal/commands/index.py b/src/pip/_internal/commands/index.py index 7267effed..f55e9e499 100644 --- a/src/pip/_internal/commands/index.py +++ b/src/pip/_internal/commands/index.py @@ -128,12 +128,12 @@ class IndexCommand(IndexGroupCommand): if not versions: raise DistributionNotFound( - "No matching distribution found for {}".format(query) + f"No matching distribution found for {query}" ) formatted_versions = [str(ver) for ver in sorted(versions, reverse=True)] latest = formatted_versions[0] - write_output("{} ({})".format(query, latest)) + write_output(f"{query} ({latest})") write_output("Available versions: {}".format(", ".join(formatted_versions))) print_dist_installation_info(query, latest) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 365764fc7..e944bb95a 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -607,12 +607,8 @@ class InstallCommand(RequirementCommand): version = package_set[project_name][0] for dependency in missing[project_name]: message = ( - "{name} {version} requires {requirement}, " + f"{project_name} {version} requires {dependency[1]}, " "which is not installed." - ).format( - name=project_name, - version=version, - requirement=dependency[1], ) parts.append(message) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 124a7ca5d..c25273d5f 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -59,8 +59,8 @@ def _disassemble_key(name: str) -> List[str]: if "." not in name: error_message = ( "Key does not contain dot separated section and key. " - "Perhaps you wanted to use 'global.{}' instead?" - ).format(name) + f"Perhaps you wanted to use 'global.{name}' instead?" + ) raise ConfigurationError(error_message) return name.split(".", 1) diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index d95fe44b3..5007a622d 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -247,10 +247,7 @@ class NoneMetadataError(PipError): def __str__(self) -> str: # Use `dist` in the error message because its stringification # includes more information, like the version and location. - return "None {} metadata found for distribution: {}".format( - self.metadata_name, - self.dist, - ) + return f"None {self.metadata_name} metadata found for distribution: {self.dist}" class UserInstallationInvalid(InstallationError): @@ -594,7 +591,7 @@ class HashMismatch(HashError): self.gots = gots def body(self) -> str: - return " {}:\n{}".format(self._requirement_name(), self._hash_comparison()) + return f" {self._requirement_name()}:\n{self._hash_comparison()}" def _hash_comparison(self) -> str: """ @@ -616,11 +613,9 @@ class HashMismatch(HashError): lines: List[str] = [] for hash_name, expecteds in self.allowed.items(): prefix = hash_then_or(hash_name) - lines.extend( - (" Expected {} {}".format(next(prefix), e)) for e in expecteds - ) + lines.extend((f" Expected {next(prefix)} {e}") for e in expecteds) lines.append( - " Got {}\n".format(self.gots[hash_name].hexdigest()) + f" Got {self.gots[hash_name].hexdigest()}\n" ) return "\n".join(lines) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 2121ca327..ec9ebc367 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -533,8 +533,8 @@ class CandidateEvaluator: ) except ValueError: raise UnsupportedWheel( - "{} is not a supported wheel for this platform. It " - "can't be sorted.".format(wheel.filename) + f"{wheel.filename} is not a supported wheel for this platform. It " + "can't be sorted." ) if self._prefer_binary: binary_preference = 1 @@ -939,9 +939,7 @@ class PackageFinder: _format_versions(best_candidate_result.iter_all()), ) - raise DistributionNotFound( - "No matching distribution found for {}".format(req) - ) + raise DistributionNotFound(f"No matching distribution found for {req}") def _should_install_candidate( candidate: Optional[InstallationCandidate], diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py index a4963aec6..9184a902a 100644 --- a/src/pip/_internal/models/candidate.py +++ b/src/pip/_internal/models/candidate.py @@ -27,8 +27,4 @@ class InstallationCandidate(KeyBasedCompareMixin): ) def __str__(self) -> str: - return "{!r} candidate (version {} at {})".format( - self.name, - self.version, - self.link, - ) + return f"{self.name!r} candidate (version {self.version} at {self.link})" diff --git a/src/pip/_internal/models/direct_url.py b/src/pip/_internal/models/direct_url.py index e219d7384..0af884bd8 100644 --- a/src/pip/_internal/models/direct_url.py +++ b/src/pip/_internal/models/direct_url.py @@ -31,9 +31,7 @@ def _get( value = d[key] if not isinstance(value, expected_type): raise DirectUrlValidationError( - "{!r} has unexpected type for {} (expected {})".format( - value, key, expected_type - ) + f"{value!r} has unexpected type for {key} (expected {expected_type})" ) return value diff --git a/src/pip/_internal/models/format_control.py b/src/pip/_internal/models/format_control.py index db3995eac..ccd11272c 100644 --- a/src/pip/_internal/models/format_control.py +++ b/src/pip/_internal/models/format_control.py @@ -33,9 +33,7 @@ class FormatControl: return all(getattr(self, k) == getattr(other, k) for k in self.__slots__) def __repr__(self) -> str: - return "{}({}, {})".format( - self.__class__.__name__, self.no_binary, self.only_binary - ) + return f"{self.__class__.__name__}({self.no_binary}, {self.only_binary})" @staticmethod def handle_mutual_excludes(value: str, target: Set[str], other: Set[str]) -> None: diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 4453519ad..73041b864 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -368,9 +368,7 @@ class Link(KeyBasedCompareMixin): else: rp = "" if self.comes_from: - return "{} (from {}){}".format( - redact_auth_from_url(self._url), self.comes_from, rp - ) + return f"{redact_auth_from_url(self._url)} (from {self.comes_from}){rp}" else: return redact_auth_from_url(str(self._url)) diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 79b82a570..d1d43541e 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -42,7 +42,7 @@ def _prepare_download( logged_url = redact_auth_from_url(url) if total_length: - logged_url = "{} ({})".format(logged_url, format_size(total_length)) + logged_url = f"{logged_url} ({format_size(total_length)})" if is_from_cache(resp): logger.info("Using cached %s", logged_url) diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 58a773059..f67180c9e 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -164,16 +164,14 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> Optional[str]: for parent_dir, dir_scripts in warn_for.items(): sorted_scripts: List[str] = sorted(dir_scripts) if len(sorted_scripts) == 1: - start_text = "script {} is".format(sorted_scripts[0]) + start_text = f"script {sorted_scripts[0]} is" else: start_text = "scripts {} are".format( ", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1] ) msg_lines.append( - "The {} installed in '{}' which is not on PATH.".format( - start_text, parent_dir - ) + f"The {start_text} installed in '{parent_dir}' which is not on PATH." ) last_line_fmt = ( @@ -321,9 +319,7 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]: scripts_to_generate.append("pip = " + pip_script) if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": - scripts_to_generate.append( - "pip{} = {}".format(sys.version_info[0], pip_script) - ) + scripts_to_generate.append(f"pip{sys.version_info[0]} = {pip_script}") scripts_to_generate.append(f"pip{get_major_minor_version()} = {pip_script}") # Delete any other versioned pip entry points @@ -336,9 +332,7 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]: scripts_to_generate.append("easy_install = " + easy_install_script) scripts_to_generate.append( - "easy_install-{} = {}".format( - get_major_minor_version(), easy_install_script - ) + f"easy_install-{get_major_minor_version()} = {easy_install_script}" ) # Delete any other versioned easy_install entry points easy_install_ep = [ @@ -408,10 +402,10 @@ class ScriptFile: class MissingCallableSuffix(InstallationError): def __init__(self, entry_point: str) -> None: super().__init__( - "Invalid script entry point: {} - A callable " + f"Invalid script entry point: {entry_point} - A callable " "suffix is required. Cf https://packaging.python.org/" "specifications/entry-points/#use-for-scripts for more " - "information.".format(entry_point) + "information." ) @@ -712,7 +706,7 @@ def req_error_context(req_description: str) -> Generator[None, None, None]: try: yield except InstallationError as e: - message = "For req: {}. {}".format(req_description, e.args[0]) + message = f"For req: {req_description}. {e.args[0]}" raise InstallationError(message) from e diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 488e76358..956717d1e 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -603,8 +603,8 @@ class RequirementPreparer: ) except NetworkConnectionError as exc: raise InstallationError( - "Could not install requirement {} because of HTTP " - "error {} for URL {}".format(req, exc, link) + f"Could not install requirement {req} because of HTTP " + f"error {exc} for URL {link}" ) else: file_path = self._downloaded[link.url] @@ -684,9 +684,9 @@ class RequirementPreparer: with indent_log(): if self.require_hashes: raise InstallationError( - "The editable requirement {} cannot be installed when " + f"The editable requirement {req} cannot be installed when " "requiring hashes, because there is no single file to " - "hash.".format(req) + "hash." ) req.ensure_has_source_dir(self.src_dir) req.update_editable() @@ -714,7 +714,7 @@ class RequirementPreparer: assert req.satisfied_by, "req should have been satisfied but isn't" assert skip_reason is not None, ( "did not get skip reason skipped but req.satisfied_by " - "is set to {}".format(req.satisfied_by) + f"is set to {req.satisfied_by}" ) logger.info( "Requirement %s: %s (%s)", skip_reason, req, req.satisfied_by.version diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index b52c9a456..7e2d0e5b8 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -462,7 +462,7 @@ def install_req_from_req_string( raise InstallationError( "Packages installed from PyPI cannot depend on packages " "which are not also hosted on PyPI.\n" - "{} depends on {} ".format(comes_from.name, req) + f"{comes_from.name} depends on {req} " ) return InstallRequirement( diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index e556be2b4..b61a219df 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -191,7 +191,7 @@ class InstallRequirement: if self.req: s = redact_auth_from_requirement(self.req) if self.link: - s += " from {}".format(redact_auth_from_url(self.link.url)) + s += f" from {redact_auth_from_url(self.link.url)}" elif self.link: s = redact_auth_from_url(self.link.url) else: @@ -221,7 +221,7 @@ class InstallRequirement: attributes = vars(self) names = sorted(attributes) - state = ("{}={!r}".format(attr, attributes[attr]) for attr in sorted(names)) + state = (f"{attr}={attributes[attr]!r}" for attr in sorted(names)) return "<{name} object: {{{state}}}>".format( name=self.__class__.__name__, state=", ".join(state), @@ -754,8 +754,8 @@ class InstallRequirement: if os.path.exists(archive_path): response = ask_path_exists( - "The file {} exists. (i)gnore, (w)ipe, " - "(b)ackup, (a)bort ".format(display_path(archive_path)), + f"The file {display_path(archive_path)} exists. (i)gnore, (w)ipe, " + "(b)ackup, (a)bort ", ("i", "w", "b", "a"), ) if response == "i": diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index 861aa4f22..3ca10098c 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -71,16 +71,16 @@ def uninstallation_paths(dist: BaseDistribution) -> Generator[str, None, None]: entries = dist.iter_declared_entries() if entries is None: - msg = "Cannot uninstall {dist}, RECORD file not found.".format(dist=dist) + msg = f"Cannot uninstall {dist}, RECORD file not found." installer = dist.installer if not installer or installer == "pip": - dep = "{}=={}".format(dist.raw_name, dist.version) + dep = f"{dist.raw_name}=={dist.version}" msg += ( " You might be able to recover from this via: " - "'pip install --force-reinstall --no-deps {}'.".format(dep) + f"'pip install --force-reinstall --no-deps {dep}'." ) else: - msg += " Hint: The package was installed by {}.".format(installer) + msg += f" Hint: The package was installed by {installer}." raise UninstallationError(msg) for entry in entries: diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index b17b7e453..5ddb848a9 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -231,9 +231,7 @@ class Resolver(BaseResolver): tags = compatibility_tags.get_supported() if requirement_set.check_supported_wheels and not wheel.supported(tags): raise InstallationError( - "{} is not a supported wheel on this platform.".format( - wheel.filename - ) + f"{wheel.filename} is not a supported wheel on this platform." ) # This next bit is really a sanity check. @@ -287,9 +285,9 @@ class Resolver(BaseResolver): ) if does_not_satisfy_constraint: raise InstallationError( - "Could not satisfy constraints for '{}': " + f"Could not satisfy constraints for '{install_req.name}': " "installation from path or url cannot be " - "constrained to a version".format(install_req.name) + "constrained to a version" ) # If we're now installing a constraint, mark the existing # object for real installation. @@ -398,9 +396,9 @@ class Resolver(BaseResolver): # "UnicodeEncodeError: 'ascii' codec can't encode character" # in Python 2 when the reason contains non-ascii characters. "The candidate selected for download or install is a " - "yanked version: {candidate}\n" - "Reason for being yanked: {reason}" - ).format(candidate=best_candidate, reason=reason) + f"yanked version: {best_candidate}\n" + f"Reason for being yanked: {reason}" + ) logger.warning(msg) return link diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 97541655f..4125cda2b 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -159,10 +159,7 @@ class _InstallRequirementBackedCandidate(Candidate): return f"{self.name} {self.version}" def __repr__(self) -> str: - return "{class_name}({link!r})".format( - class_name=self.__class__.__name__, - link=str(self._link), - ) + return f"{self.__class__.__name__}({str(self._link)!r})" def __hash__(self) -> int: return hash((self.__class__, self._link)) @@ -354,10 +351,7 @@ class AlreadyInstalledCandidate(Candidate): return str(self.dist) def __repr__(self) -> str: - return "{class_name}({distribution!r})".format( - class_name=self.__class__.__name__, - distribution=self.dist, - ) + return f"{self.__class__.__name__}({self.dist!r})" def __hash__(self) -> int: return hash((self.__class__, self.name, self.version)) @@ -455,11 +449,7 @@ class ExtrasCandidate(Candidate): return "{}[{}] {}".format(name, ",".join(self.extras), rest) def __repr__(self) -> str: - return "{class_name}(base={base!r}, extras={extras!r})".format( - class_name=self.__class__.__name__, - base=self.base, - extras=self.extras, - ) + return f"{self.__class__.__name__}(base={self.base!r}, extras={self.extras!r})" def __hash__(self) -> int: return hash((self.base, self.extras)) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 38c199448..97137c997 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -753,8 +753,8 @@ class Factory: info = "the requested packages" msg = ( - "Cannot install {} because these package versions " - "have conflicting dependencies.".format(info) + f"Cannot install {info} because these package versions " + "have conflicting dependencies." ) logger.critical(msg) msg = "\nThe conflict is caused by:" diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index 7d1e7bfdd..4af4a9f25 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -15,10 +15,7 @@ class ExplicitRequirement(Requirement): return str(self.candidate) def __repr__(self) -> str: - return "{class_name}({candidate!r})".format( - class_name=self.__class__.__name__, - candidate=self.candidate, - ) + return f"{self.__class__.__name__}({self.candidate!r})" @property def project_name(self) -> NormalizedName: @@ -50,10 +47,7 @@ class SpecifierRequirement(Requirement): return str(self._ireq.req) def __repr__(self) -> str: - return "{class_name}({requirement!r})".format( - class_name=self.__class__.__name__, - requirement=str(self._ireq.req), - ) + return f"{self.__class__.__name__}({str(self._ireq.req)!r})" @property def project_name(self) -> NormalizedName: @@ -116,10 +110,7 @@ class RequiresPythonRequirement(Requirement): return f"Python {self.specifier}" def __repr__(self) -> str: - return "{class_name}({specifier!r})".format( - class_name=self.__class__.__name__, - specifier=str(self.specifier), - ) + return f"{self.__class__.__name__}({str(self.specifier)!r})" @property def project_name(self) -> NormalizedName: @@ -155,10 +146,7 @@ class UnsatisfiableRequirement(Requirement): return f"{self._name} (unavailable)" def __repr__(self) -> str: - return "{class_name}({name!r})".format( - class_name=self.__class__.__name__, - name=str(self._name), - ) + return f"{self.__class__.__name__}({str(self._name)!r})" @property def project_name(self) -> NormalizedName: diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 78060e864..42a8536cd 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -77,11 +77,7 @@ def get_pip_version() -> str: pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..") pip_pkg_dir = os.path.abspath(pip_pkg_dir) - return "pip {} from {} (python {})".format( - __version__, - pip_pkg_dir, - get_major_minor_version(), - ) + return f"pip {__version__} from {pip_pkg_dir} (python {get_major_minor_version()})" def normalize_version_info(py_version_info: Tuple[int, ...]) -> Tuple[int, int, int]: @@ -279,13 +275,13 @@ def strtobool(val: str) -> int: def format_size(bytes: float) -> str: if bytes > 1000 * 1000: - return "{:.1f} MB".format(bytes / 1000.0 / 1000) + return f"{bytes / 1000.0 / 1000:.1f} MB" elif bytes > 10 * 1000: - return "{} kB".format(int(bytes / 1000)) + return f"{int(bytes / 1000)} kB" elif bytes > 1000: - return "{:.1f} kB".format(bytes / 1000.0) + return f"{bytes / 1000.0:.1f} kB" else: - return "{} bytes".format(int(bytes)) + return f"{int(bytes)} bytes" def tabulate(rows: Iterable[Iterable[Any]]) -> Tuple[List[str], List[int]]: @@ -522,9 +518,7 @@ def redact_netloc(netloc: str) -> str: else: user = urllib.parse.quote(user) password = ":****" - return "{user}{password}@{netloc}".format( - user=user, password=password, netloc=netloc - ) + return f"{user}{password}@{netloc}" def _transform_url( @@ -592,7 +586,7 @@ class HiddenText: self.redacted = redacted def __repr__(self) -> str: - return "".format(str(self)) + return f"" def __str__(self) -> str: return self.redacted diff --git a/src/pip/_internal/utils/wheel.py b/src/pip/_internal/utils/wheel.py index e5e3f34ed..3551f8f19 100644 --- a/src/pip/_internal/utils/wheel.py +++ b/src/pip/_internal/utils/wheel.py @@ -28,7 +28,7 @@ def parse_wheel(wheel_zip: ZipFile, name: str) -> Tuple[str, Message]: metadata = wheel_metadata(wheel_zip, info_dir) version = wheel_version(metadata) except UnsupportedWheel as e: - raise UnsupportedWheel("{} has an invalid wheel, {}".format(name, str(e))) + raise UnsupportedWheel(f"{name} has an invalid wheel, {str(e)}") check_compatibility(version, name) @@ -60,9 +60,7 @@ def wheel_dist_info_dir(source: ZipFile, name: str) -> str: canonical_name = canonicalize_name(name) if not info_dir_name.startswith(canonical_name): raise UnsupportedWheel( - ".dist-info directory {!r} does not start with {!r}".format( - info_dir, canonical_name - ) + f".dist-info directory {info_dir!r} does not start with {canonical_name!r}" ) return info_dir diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 02bbf68e7..46ca2799b 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -405,9 +405,9 @@ class VersionControl: scheme, netloc, path, query, frag = urllib.parse.urlsplit(url) if "+" not in scheme: raise ValueError( - "Sorry, {!r} is a malformed VCS url. " + f"Sorry, {url!r} is a malformed VCS url. " "The format is +://, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" ) # Remove the vcs prefix. scheme = scheme.split("+", 1)[1] @@ -417,9 +417,9 @@ class VersionControl: path, rev = path.rsplit("@", 1) if not rev: raise InstallationError( - "The URL {!r} has an empty revision (after @) " + f"The URL {url!r} has an empty revision (after @) " "which is not supported. Include a revision after @ " - "or remove @ from the URL.".format(url) + "or remove @ from the URL." ) url = urllib.parse.urlunsplit((scheme, netloc, path, query, "")) return url, rev, user_pass @@ -566,7 +566,7 @@ class VersionControl: self.name, url, ) - response = ask_path_exists("What to do? {}".format(prompt[0]), prompt[1]) + response = ask_path_exists(f"What to do? {prompt[0]}", prompt[1]) if response == "a": sys.exit(-1) diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index 60d75dd18..b1debe349 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -140,15 +140,15 @@ def _verify_one(req: InstallRequirement, wheel_path: str) -> None: w = Wheel(os.path.basename(wheel_path)) if canonicalize_name(w.name) != canonical_name: raise InvalidWheelFilename( - "Wheel has unexpected file name: expected {!r}, " - "got {!r}".format(canonical_name, w.name), + f"Wheel has unexpected file name: expected {canonical_name!r}, " + f"got {w.name!r}", ) dist = get_wheel_distribution(FilesystemWheel(wheel_path), canonical_name) dist_verstr = str(dist.version) if canonicalize_version(dist_verstr) != canonicalize_version(w.version): raise InvalidWheelFilename( - "Wheel has unexpected file name: expected {!r}, " - "got {!r}".format(dist_verstr, w.version), + f"Wheel has unexpected file name: expected {dist_verstr!r}, " + f"got {w.version!r}", ) metadata_version_value = dist.metadata_version if metadata_version_value is None: @@ -160,8 +160,7 @@ def _verify_one(req: InstallRequirement, wheel_path: str) -> None: raise UnsupportedWheel(msg) if metadata_version >= Version("1.2") and not isinstance(dist.version, Version): raise UnsupportedWheel( - "Metadata 1.2 mandates PEP 440 version, " - "but {!r} is not".format(dist_verstr) + f"Metadata 1.2 mandates PEP 440 version, but {dist_verstr!r} is not" ) diff --git a/tests/conftest.py b/tests/conftest.py index 6ae2e6d62..8e498abd0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -141,7 +141,7 @@ def pytest_collection_modifyitems(config: Config, items: List[pytest.Function]) if "script" in item.fixturenames: raise RuntimeError( "Cannot use the ``script`` funcarg in a unit test: " - "(filename = {}, item = {})".format(module_path, item) + f"(filename = {module_path}, item = {item})" ) else: raise RuntimeError(f"Unknown test type (filename = {module_path})") diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index a1b69b721..3c3f45d51 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -23,7 +23,7 @@ def test_entrypoints_work(entrypoint: str, script: PipTestEnvironment) -> None: fake_pkg.mkdir() fake_pkg.joinpath("setup.py").write_text( dedent( - """ + f""" from setuptools import setup setup( @@ -31,13 +31,11 @@ def test_entrypoints_work(entrypoint: str, script: PipTestEnvironment) -> None: version="0.1.0", entry_points={{ "console_scripts": [ - {!r} + {entrypoint!r} ] }} ) - """.format( - entrypoint - ) + """ ) ) diff --git a/tests/functional/test_completion.py b/tests/functional/test_completion.py index 2aa861aac..4be033583 100644 --- a/tests/functional/test_completion.py +++ b/tests/functional/test_completion.py @@ -400,7 +400,7 @@ def test_completion_path_after_option( def test_completion_uses_same_executable_name( autocomplete_script: PipTestEnvironment, flag: str, deprecated_python: bool ) -> None: - executable_name = "pip{}".format(sys.version_info[0]) + executable_name = f"pip{sys.version_info[0]}" # Deprecated python versions produce an extra deprecation warning result = autocomplete_script.run( executable_name, diff --git a/tests/functional/test_debug.py b/tests/functional/test_debug.py index 77cd732f9..77d4bea33 100644 --- a/tests/functional/test_debug.py +++ b/tests/functional/test_debug.py @@ -68,7 +68,7 @@ def test_debug__tags(script: PipTestEnvironment, args: List[str]) -> None: stdout = result.stdout tags = compatibility_tags.get_supported() - expected_tag_header = "Compatible tags: {}".format(len(tags)) + expected_tag_header = f"Compatible tags: {len(tags)}" assert expected_tag_header in stdout show_verbose_note = "--verbose" not in args diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 9a5937df3..b2fd1d629 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -166,13 +166,11 @@ def test_freeze_with_invalid_names(script: PipTestEnvironment) -> None: with open(egg_info_path, "w") as egg_info_file: egg_info_file.write( textwrap.dedent( - """\ + f"""\ Metadata-Version: 1.0 - Name: {} + Name: {pkgname} Version: 1.0 - """.format( - pkgname - ) + """ ) ) @@ -221,12 +219,10 @@ def test_freeze_editable_not_vcs(script: PipTestEnvironment) -> None: # We need to apply os.path.normcase() to the path since that is what # the freeze code does. expected = textwrap.dedent( - """\ + f"""\ ...# Editable install with no version control (version-pkg==0.1) - -e {} - ...""".format( - os.path.normcase(pkg_path) - ) + -e {os.path.normcase(pkg_path)} + ...""" ) _check_output(result.stdout, expected) @@ -248,12 +244,10 @@ def test_freeze_editable_git_with_no_remote( # We need to apply os.path.normcase() to the path since that is what # the freeze code does. expected = textwrap.dedent( - """\ + f"""\ ...# Editable Git install with no remote (version-pkg==0.1) - -e {} - ...""".format( - os.path.normcase(pkg_path) - ) + -e {os.path.normcase(pkg_path)} + ...""" ) _check_output(result.stdout, expected) @@ -653,9 +647,9 @@ def test_freeze_with_requirement_option_file_url_egg_not_installed( expect_stderr=True, ) expected_err = ( - "WARNING: Requirement file [requirements.txt] contains {}, " + f"WARNING: Requirement file [requirements.txt] contains {url}, " "but package 'Does.Not-Exist' is not installed\n" - ).format(url) + ) if deprecated_python: assert expected_err in result.stderr else: diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 140061a17..b18fabc84 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -106,10 +106,10 @@ def test_pep518_refuses_conflicting_requires( assert ( result.returncode != 0 and ( - "Some build dependencies for {url} conflict " + f"Some build dependencies for {project_dir.as_uri()} conflict " "with PEP 517/518 supported " "requirements: setuptools==1.0 is incompatible with " - "setuptools>=40.8.0.".format(url=project_dir.as_uri()) + "setuptools>=40.8.0." ) in result.stderr ), str(result) @@ -595,8 +595,8 @@ def test_hashed_install_success( with requirements_file( "simple2==1.0 --hash=sha256:9336af72ca661e6336eb87bc7de3e8844d853e" "3848c2b9bbd2e8bf01db88c2c7\n" - "{simple} --hash=sha256:393043e672415891885c9a2a0929b1af95fb866d6c" - "a016b42d2e6ce53619b653".format(simple=file_url), + f"{file_url} --hash=sha256:393043e672415891885c9a2a0929b1af95fb866d6c" + "a016b42d2e6ce53619b653", tmpdir, ) as reqs_file: script.pip_install_local("-r", reqs_file.resolve()) @@ -1735,7 +1735,7 @@ def test_install_builds_wheels(script: PipTestEnvironment, data: TestData) -> No # into the cache assert wheels != [], str(res) assert wheels == [ - "Upper-2.0-py{}-none-any.whl".format(sys.version_info[0]), + f"Upper-2.0-py{sys.version_info[0]}-none-any.whl", ] @@ -2387,7 +2387,7 @@ def test_install_verify_package_name_normalization( assert "Successfully installed simple-package" in result.stdout result = script.pip("install", package_name) - assert "Requirement already satisfied: {}".format(package_name) in result.stdout + assert f"Requirement already satisfied: {package_name}" in result.stdout def test_install_logs_pip_version_in_debug( diff --git a/tests/functional/test_install_config.py b/tests/functional/test_install_config.py index ecaf2f705..7f418067f 100644 --- a/tests/functional/test_install_config.py +++ b/tests/functional/test_install_config.py @@ -184,12 +184,10 @@ def test_config_file_override_stack( config_file.write_text( textwrap.dedent( - """\ + f"""\ [global] - index-url = {}/simple1 - """.format( - base_address - ) + index-url = {base_address}/simple1 + """ ) ) script.pip("install", "-vvv", "INITools", expect_error=True) @@ -197,14 +195,12 @@ def test_config_file_override_stack( config_file.write_text( textwrap.dedent( - """\ + f"""\ [global] - index-url = {address}/simple1 + index-url = {base_address}/simple1 [install] - index-url = {address}/simple2 - """.format( - address=base_address - ) + index-url = {base_address}/simple2 + """ ) ) script.pip("install", "-vvv", "INITools", expect_error=True) diff --git a/tests/functional/test_install_index.py b/tests/functional/test_install_index.py index b73e28f47..72b0b9db7 100644 --- a/tests/functional/test_install_index.py +++ b/tests/functional/test_install_index.py @@ -41,13 +41,11 @@ def test_find_links_requirements_file_relative_path( """Test find-links as a relative path to a reqs file.""" script.scratch_path.joinpath("test-req.txt").write_text( textwrap.dedent( - """ + f""" --no-index - --find-links={} + --find-links={data.packages.as_posix()} parent==0.1 - """.format( - data.packages.as_posix() - ) + """ ) ) result = script.pip( diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index c21b9ba83..c13258178 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -95,7 +95,7 @@ def test_requirements_file(script: PipTestEnvironment) -> None: result.did_create(script.site_packages / "INITools-0.2.dist-info") result.did_create(script.site_packages / "initools") assert result.files_created[script.site_packages / other_lib_name].dir - fn = "{}-{}.dist-info".format(other_lib_name, other_lib_version) + fn = f"{other_lib_name}-{other_lib_version}.dist-info" assert result.files_created[script.site_packages / fn].dir @@ -260,13 +260,13 @@ def test_respect_order_in_requirements_file( assert ( "parent" in downloaded[0] - ), 'First download should be "parent" but was "{}"'.format(downloaded[0]) + ), f'First download should be "parent" but was "{downloaded[0]}"' assert ( "child" in downloaded[1] - ), 'Second download should be "child" but was "{}"'.format(downloaded[1]) + ), f'Second download should be "child" but was "{downloaded[1]}"' assert ( "simple" in downloaded[2] - ), 'Third download should be "simple" but was "{}"'.format(downloaded[2]) + ), f'Third download should be "simple" but was "{downloaded[2]}"' def test_install_local_editable_with_extras( diff --git a/tests/functional/test_install_wheel.py b/tests/functional/test_install_wheel.py index 4221ae76a..7e7aeaf7a 100644 --- a/tests/functional/test_install_wheel.py +++ b/tests/functional/test_install_wheel.py @@ -169,9 +169,9 @@ def get_header_scheme_path_for_script( ) -> Path: command = ( "from pip._internal.locations import get_scheme;" - "scheme = get_scheme({!r});" + f"scheme = get_scheme({dist_name!r});" "print(scheme.headers);" - ).format(dist_name) + ) result = script.run("python", "-c", command).stdout return Path(result.strip()) diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index b5945edf8..df82713d0 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -1185,7 +1185,7 @@ def test_new_resolver_presents_messages_when_backtracking_a_lot( for index in range(1, N + 1): A_version = f"{index}.0.0" B_version = f"{index}.0.0" - C_version = "{index_minus_one}.0.0".format(index_minus_one=index - 1) + C_version = f"{index - 1}.0.0" depends = ["B == " + B_version] if index != 1: diff --git a/tests/functional/test_new_resolver_errors.py b/tests/functional/test_new_resolver_errors.py index 623041312..5976de52e 100644 --- a/tests/functional/test_new_resolver_errors.py +++ b/tests/functional/test_new_resolver_errors.py @@ -71,8 +71,8 @@ def test_new_resolver_conflict_constraints_file( def test_new_resolver_requires_python_error(script: PipTestEnvironment) -> None: - compatible_python = ">={0.major}.{0.minor}".format(sys.version_info) - incompatible_python = "<{0.major}.{0.minor}".format(sys.version_info) + compatible_python = f">={sys.version_info.major}.{sys.version_info.minor}" + incompatible_python = f"<{sys.version_info.major}.{sys.version_info.minor}" pkga = create_test_package_with_setup( script, @@ -99,7 +99,7 @@ def test_new_resolver_requires_python_error(script: PipTestEnvironment) -> None: def test_new_resolver_checks_requires_python_before_dependencies( script: PipTestEnvironment, ) -> None: - incompatible_python = "<{0.major}.{0.minor}".format(sys.version_info) + incompatible_python = f"<{sys.version_info.major}.{sys.version_info.minor}" pkg_dep = create_basic_wheel_for_package( script, diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 6db2efd0e..d26def14a 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -24,18 +24,11 @@ def _create_find_links(script: PipTestEnvironment) -> _FindLinks: index_html = script.scratch_path / "index.html" index_html.write_text( - """ + f""" - {sdist_path.stem} - {wheel_path.stem} - """.format( - sdist_url=sdist_path.as_uri(), - sdist_hash=sdist_hash, - sdist_path=sdist_path, - wheel_url=wheel_path.as_uri(), - wheel_hash=wheel_hash, - wheel_path=wheel_path, - ).strip() + {sdist_path.stem} + {wheel_path.stem} + """.strip() ) return _FindLinks(index_html, sdist_hash, wheel_hash) @@ -99,9 +92,7 @@ def test_new_resolver_hash_intersect_from_constraint( constraints_txt = script.scratch_path / "constraints.txt" constraints_txt.write_text( - "base==0.1.0 --hash=sha256:{sdist_hash}".format( - sdist_hash=find_links.sdist_hash, - ), + f"base==0.1.0 --hash=sha256:{find_links.sdist_hash}", ) requirements_txt = script.scratch_path / "requirements.txt" requirements_txt.write_text( @@ -200,13 +191,10 @@ def test_new_resolver_hash_intersect_empty_from_constraint( constraints_txt = script.scratch_path / "constraints.txt" constraints_txt.write_text( - """ - base==0.1.0 --hash=sha256:{sdist_hash} - base==0.1.0 --hash=sha256:{wheel_hash} - """.format( - sdist_hash=find_links.sdist_hash, - wheel_hash=find_links.wheel_hash, - ), + f""" + base==0.1.0 --hash=sha256:{find_links.sdist_hash} + base==0.1.0 --hash=sha256:{find_links.wheel_hash} + """, ) result = script.pip( @@ -240,19 +228,15 @@ def test_new_resolver_hash_requirement_and_url_constraint_can_succeed( requirements_txt = script.scratch_path / "requirements.txt" requirements_txt.write_text( - """ + f""" base==0.1.0 --hash=sha256:{wheel_hash} - """.format( - wheel_hash=wheel_hash, - ), + """, ) constraints_txt = script.scratch_path / "constraints.txt" - constraint_text = "base @ {wheel_url}\n".format(wheel_url=wheel_path.as_uri()) + constraint_text = f"base @ {wheel_path.as_uri()}\n" if constrain_by_hash: - constraint_text += "base==0.1.0 --hash=sha256:{wheel_hash}\n".format( - wheel_hash=wheel_hash, - ) + constraint_text += f"base==0.1.0 --hash=sha256:{wheel_hash}\n" constraints_txt.write_text(constraint_text) script.pip( @@ -280,19 +264,15 @@ def test_new_resolver_hash_requirement_and_url_constraint_can_fail( requirements_txt = script.scratch_path / "requirements.txt" requirements_txt.write_text( - """ + f""" base==0.1.0 --hash=sha256:{other_hash} - """.format( - other_hash=other_hash, - ), + """, ) constraints_txt = script.scratch_path / "constraints.txt" - constraint_text = "base @ {wheel_url}\n".format(wheel_url=wheel_path.as_uri()) + constraint_text = f"base @ {wheel_path.as_uri()}\n" if constrain_by_hash: - constraint_text += "base==0.1.0 --hash=sha256:{other_hash}\n".format( - other_hash=other_hash, - ) + constraint_text += f"base==0.1.0 --hash=sha256:{other_hash}\n" constraints_txt.write_text(constraint_text) result = script.pip( @@ -343,17 +323,12 @@ def test_new_resolver_hash_with_extras(script: PipTestEnvironment) -> None: requirements_txt = script.scratch_path / "requirements.txt" requirements_txt.write_text( - """ + f""" child[extra]==0.1.0 --hash=sha256:{child_hash} parent_with_extra==0.1.0 --hash=sha256:{parent_with_extra_hash} parent_without_extra==0.1.0 --hash=sha256:{parent_without_extra_hash} extra==0.1.0 --hash=sha256:{extra_hash} - """.format( - child_hash=child_hash, - parent_with_extra_hash=parent_with_extra_hash, - parent_without_extra_hash=parent_without_extra_hash, - extra_hash=extra_hash, - ), + """, ) script.pip( diff --git a/tests/functional/test_new_resolver_target.py b/tests/functional/test_new_resolver_target.py index 811ae935a..a81cfe5e8 100644 --- a/tests/functional/test_new_resolver_target.py +++ b/tests/functional/test_new_resolver_target.py @@ -58,12 +58,7 @@ def test_new_resolver_target_checks_compatibility_failure( if platform: args += ["--platform", platform] - args_tag = "{}{}-{}-{}".format( - implementation, - python_version, - abi, - platform, - ) + args_tag = f"{implementation}{python_version}-{abi}-{platform}" wheel_tag_matches = args_tag == fake_wheel_tag result = script.pip(*args, expect_error=(not wheel_tag_matches)) diff --git a/tests/functional/test_pep517.py b/tests/functional/test_pep517.py index a642a3f8b..78a6c2bbc 100644 --- a/tests/functional/test_pep517.py +++ b/tests/functional/test_pep517.py @@ -159,9 +159,9 @@ def test_conflicting_pep517_backend_requirements( expect_error=True, ) msg = ( - "Some build dependencies for {url} conflict with the backend " + f"Some build dependencies for {project_dir.as_uri()} conflict with the backend " "dependencies: simplewheel==1.0 is incompatible with " - "simplewheel==2.0.".format(url=project_dir.as_uri()) + "simplewheel==2.0." ) assert result.returncode != 0 and msg in result.stderr, str(result) @@ -205,8 +205,8 @@ def test_validate_missing_pep517_backend_requirements( expect_error=True, ) msg = ( - "Some build dependencies for {url} are missing: " - "'simplewheel==1.0', 'test_backend'.".format(url=project_dir.as_uri()) + f"Some build dependencies for {project_dir.as_uri()} are missing: " + "'simplewheel==1.0', 'test_backend'." ) assert result.returncode != 0 and msg in result.stderr, str(result) @@ -231,9 +231,9 @@ def test_validate_conflicting_pep517_backend_requirements( expect_error=True, ) msg = ( - "Some build dependencies for {url} conflict with the backend " + f"Some build dependencies for {project_dir.as_uri()} conflict with the backend " "dependencies: simplewheel==2.0 is incompatible with " - "simplewheel==1.0.".format(url=project_dir.as_uri()) + "simplewheel==1.0." ) assert result.returncode != 0 and msg in result.stderr, str(result) diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index be7fe4c33..69e340a56 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -604,9 +604,7 @@ def test_uninstall_without_record_fails( "simple.dist==0.1'." ) elif installer: - expected_error_message += " Hint: The package was installed by {}.".format( - installer - ) + expected_error_message += f" Hint: The package was installed by {installer}." assert result2.stderr.rstrip() == expected_error_message assert_all_changes(result.files_after, result2, ignore_changes) diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index 042f58246..b1183fc83 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -59,9 +59,7 @@ def test_pip_wheel_success(script: PipTestEnvironment, data: TestData) -> None: wheel_file_path = script.scratch / wheel_file_name assert re.search( r"Created wheel for simple: " - r"filename={filename} size=\d+ sha256=[A-Fa-f0-9]{{64}}".format( - filename=re.escape(wheel_file_name) - ), + rf"filename={re.escape(wheel_file_name)} size=\d+ sha256=[A-Fa-f0-9]{{64}}", result.stdout, ) assert re.search(r"^\s+Stored in directory: ", result.stdout, re.M) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index d27c02e25..f14837e24 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -747,7 +747,7 @@ class PipTestEnvironment(TestFileEnvironment): for val in json.loads(ret.stdout) } expected = {(canonicalize_name(k), v) for k, v in kwargs.items()} - assert expected <= installed, "{!r} not all in {!r}".format(expected, installed) + assert expected <= installed, f"{expected!r} not all in {installed!r}" def assert_not_installed(self, *args: str) -> None: ret = self.pip("list", "--format=json") @@ -755,9 +755,7 @@ class PipTestEnvironment(TestFileEnvironment): # None of the given names should be listed as installed, i.e. their # intersection should be empty. expected = {canonicalize_name(k) for k in args} - assert not (expected & installed), "{!r} contained in {!r}".format( - expected, installed - ) + assert not (expected & installed), f"{expected!r} contained in {installed!r}" # FIXME ScriptTest does something similar, but only within a single @@ -1028,7 +1026,7 @@ def _create_test_package_with_srcdir( pkg_path.joinpath("__init__.py").write_text("") subdir_path.joinpath("setup.py").write_text( textwrap.dedent( - """ + f""" from setuptools import setup, find_packages setup( name="{name}", @@ -1036,9 +1034,7 @@ def _create_test_package_with_srcdir( packages=find_packages(), package_dir={{"": "src"}}, ) - """.format( - name=name - ) + """ ) ) return _vcs_add(dir_path, version_pkg_path, vcs) @@ -1052,7 +1048,7 @@ def _create_test_package( _create_main_file(version_pkg_path, name=name, output="0.1") version_pkg_path.joinpath("setup.py").write_text( textwrap.dedent( - """ + f""" from setuptools import setup, find_packages setup( name="{name}", @@ -1061,9 +1057,7 @@ def _create_test_package( py_modules=["{name}"], entry_points=dict(console_scripts=["{name}={name}:main"]), ) - """.format( - name=name - ) + """ ) ) return _vcs_add(dir_path, version_pkg_path, vcs) @@ -1137,7 +1131,7 @@ def urlsafe_b64encode_nopad(data: bytes) -> str: def create_really_basic_wheel(name: str, version: str) -> bytes: def digest(contents: bytes) -> str: - return "sha256={}".format(urlsafe_b64encode_nopad(sha256(contents).digest())) + return f"sha256={urlsafe_b64encode_nopad(sha256(contents).digest())}" def add_file(path: str, text: str) -> None: contents = text.encode("utf-8") @@ -1153,13 +1147,11 @@ def create_really_basic_wheel(name: str, version: str) -> bytes: add_file( f"{dist_info}/METADATA", dedent( - """\ + f"""\ Metadata-Version: 2.1 - Name: {} - Version: {} - """.format( - name, version - ) + Name: {name} + Version: {version} + """ ), ) z.writestr(record_path, "\n".join(",".join(r) for r in records)) diff --git a/tests/lib/local_repos.py b/tests/lib/local_repos.py index a04d1d0fe..a8cf4aa6c 100644 --- a/tests/lib/local_repos.py +++ b/tests/lib/local_repos.py @@ -56,7 +56,7 @@ def local_checkout( assert vcs_backend is not None vcs_backend.obtain(repo_url_path, url=hide_url(remote_repo), verbosity=0) - return "{}+{}".format(vcs_name, Path(repo_url_path).as_uri()) + return f"{vcs_name}+{Path(repo_url_path).as_uri()}" def local_repo(remote_repo: str, temp_path: Path) -> str: diff --git a/tests/lib/server.py b/tests/lib/server.py index 1048a173d..96ac5930d 100644 --- a/tests/lib/server.py +++ b/tests/lib/server.py @@ -152,7 +152,7 @@ def html5_page(text: str) -> str: def package_page(spec: Dict[str, str]) -> "WSGIApplication": def link(name: str, value: str) -> str: - return '{}'.format(value, name) + return f'{name}' links = "".join(link(*kv) for kv in spec.items()) return text_html_response(html5_page(links)) diff --git a/tests/lib/test_lib.py b/tests/lib/test_lib.py index a541a0a20..c01c1beb8 100644 --- a/tests/lib/test_lib.py +++ b/tests/lib/test_lib.py @@ -107,8 +107,8 @@ class TestPipTestEnvironment: """ command = ( "import logging; logging.basicConfig(level='INFO'); " - "logging.getLogger().info('sub: {}', 'foo')" - ).format(sub_string) + f"logging.getLogger().info('sub: {sub_string}', 'foo')" + ) args = [sys.executable, "-c", command] script.run(*args, **kwargs) diff --git a/tests/lib/wheel.py b/tests/lib/wheel.py index f2ddfd3b7..a4efe53b4 100644 --- a/tests/lib/wheel.py +++ b/tests/lib/wheel.py @@ -190,7 +190,7 @@ def urlsafe_b64encode_nopad(data: bytes) -> str: def digest(contents: bytes) -> str: - return "sha256={}".format(urlsafe_b64encode_nopad(sha256(contents).digest())) + return f"sha256={urlsafe_b64encode_nopad(sha256(contents).digest())}" def record_file_maker_wrapper( diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index 3c8b81de4..dae083efe 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -119,8 +119,8 @@ def test_get_index_content_invalid_content_type_archive( assert ( "pip._internal.index.collector", logging.WARNING, - "Skipping page {} because it looks like an archive, and cannot " - "be checked by a HTTP HEAD request.".format(url), + f"Skipping page {url} because it looks like an archive, and cannot " + "be checked by a HTTP HEAD request.", ) in caplog.record_tuples @@ -417,8 +417,8 @@ def _test_parse_links_data_attribute( html = ( "" '' - "{}" - ).format(anchor_html) + f"{anchor_html}" + ) html_bytes = html.encode("utf-8") page = IndexContent( html_bytes, @@ -764,8 +764,8 @@ def test_get_index_content_invalid_scheme( ( "pip._internal.index.collector", logging.WARNING, - "Cannot look at {} URL {} because it does not support " - "lookup as web pages.".format(vcs_scheme, url), + f"Cannot look at {vcs_scheme} URL {url} because it does not support " + "lookup as web pages.", ), ] diff --git a/tests/unit/test_link.py b/tests/unit/test_link.py index 311be5888..a379d877b 100644 --- a/tests/unit/test_link.py +++ b/tests/unit/test_link.py @@ -143,10 +143,7 @@ class TestLink: def test_is_hash_allowed( self, hash_name: str, hex_digest: str, expected: bool ) -> None: - url = "https://example.com/wheel.whl#{hash_name}={hex_digest}".format( - hash_name=hash_name, - hex_digest=hex_digest, - ) + url = f"https://example.com/wheel.whl#{hash_name}={hex_digest}" link = Link(url) hashes_data = { "sha512": [128 * "a", 128 * "b"], diff --git a/tests/unit/test_network_utils.py b/tests/unit/test_network_utils.py index cdc10b2ba..380d5741f 100644 --- a/tests/unit/test_network_utils.py +++ b/tests/unit/test_network_utils.py @@ -21,8 +21,8 @@ def test_raise_for_status_raises_exception(status_code: int, error_type: str) -> with pytest.raises(NetworkConnectionError) as excinfo: raise_for_status(resp) assert str(excinfo.value) == ( - "{} {}: Network Error for url:" - " http://www.example.com/whatever.tgz".format(status_code, error_type) + f"{status_code} {error_type}: Network Error for url:" + " http://www.example.com/whatever.tgz" ) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 863f070af..5e3c640a5 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -235,8 +235,8 @@ class TestRequirementSet: r"file \(line 1\)\)\n" r"Can't verify hashes for these file:// requirements because " r"they point to directories:\n" - r" file://.*{sep}data{sep}packages{sep}FSPkg " - r"\(from -r file \(line 2\)\)".format(sep=sep) + rf" file://.*{sep}data{sep}packages{sep}FSPkg " + r"\(from -r file \(line 2\)\)" ), ): resolver.resolve(reqset.all_requirements, True) diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 7a196eb8d..94ccfb98d 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -297,7 +297,7 @@ class TestProcessLine: def test_yield_line_constraint(self, line_processor: LineProcessor) -> None: line = "SomeProject" filename = "filename" - comes_from = "-c {} (line {})".format(filename, 1) + comes_from = f"-c {filename} (line {1})" req = install_req_from_line(line, comes_from=comes_from, constraint=True) found_req = line_processor(line, filename, 1, constraint=True)[0] assert repr(found_req) == repr(req) @@ -326,7 +326,7 @@ class TestProcessLine: url = "git+https://url#egg=SomeProject" line = f"-e {url}" filename = "filename" - comes_from = "-c {} (line {})".format(filename, 1) + comes_from = f"-c {filename} (line {1})" req = install_req_from_editable(url, comes_from=comes_from, constraint=True) found_req = line_processor(line, filename, 1, constraint=True)[0] assert repr(found_req) == repr(req) @@ -873,12 +873,10 @@ class TestParseRequirements: ) -> None: global_option = "--dry-run" - content = """ + content = f""" --only-binary :all: INITools==2.0 --global-option="{global_option}" - """.format( - global_option=global_option - ) + """ with requirements_file(content, tmpdir) as reqs_file: req = next( diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 8b9d1a58a..133e59222 100644 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -261,8 +261,8 @@ class TestCheckDistRequiresPython: ignore_requires_python=False, ) assert str(exc.value) == ( - "None {} metadata found for distribution: " - "".format(metadata_name) + f"None {metadata_name} metadata found for distribution: " + "" ) diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 6d6d1a3dc..33329fbce 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -102,15 +102,13 @@ def test_get_legacy_build_wheel_path__multiple_names( ], ) def test_get_entrypoints(tmp_path: pathlib.Path, console_scripts: str) -> None: - entry_points_text = """ + entry_points_text = f""" [console_scripts] - {} + {console_scripts} [section] common:one = module:func common:two = module:other_func - """.format( - console_scripts - ) + """ distribution = make_wheel( "simple", diff --git a/tools/release/check_version.py b/tools/release/check_version.py index e89d1b5ba..de3658faa 100644 --- a/tools/release/check_version.py +++ b/tools/release/check_version.py @@ -27,7 +27,7 @@ def is_this_a_good_version_number(string: str) -> Optional[str]: expected_major = datetime.now().year % 100 if len(release) not in [2, 3]: - return "Not of the form: {0}.N or {0}.N.P".format(expected_major) + return f"Not of the form: {expected_major}.N or {expected_major}.N.P" return None From 2a0acb595c4b6394f1f3a4e7ef034ac2e3e8c17e Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Tue, 7 Nov 2023 04:39:01 -0500 Subject: [PATCH 156/156] Update and provide fixes for mypy pre-commit (#12389) * Update mypy to 1.6.1 * Fix mypy "Source file found twice under different module names" error * Ignore type of intialized abstract class in tests * Use more specific type ignore method-assign * Type ignore for message.get_all * Remove unused type ignore * Add SizedBuffer type for xmlrpc.client.Transport subclass * Add Self type for RequestHandlerClass in test * Add type ignore for shutil.rmtree onexc handler * Quote SizedBuffer * Add news entry * Remove no longer correct comment * Update self import * Also ignore type onerror=handler * Update news entry * Update news entry --- .pre-commit-config.yaml | 16 ++++++++-------- news/12389.bugfix.rst | 1 + src/pip/_internal/locations/_distutils.py | 3 +-- src/pip/_internal/metadata/_json.py | 4 ++-- src/pip/_internal/network/xmlrpc.py | 4 +++- src/pip/_internal/utils/misc.py | 4 ++-- tests/conftest.py | 6 +++++- tests/lib/configuration_helpers.py | 2 +- tests/lib/test_wheel.py | 10 +++++----- tests/unit/metadata/test_metadata.py | 6 +++--- tests/unit/test_base_command.py | 6 +++--- tests/unit/test_configuration.py | 6 +++--- tests/unit/test_resolution_legacy_resolver.py | 2 +- tools/__init__.py | 0 14 files changed, 38 insertions(+), 32 deletions(-) create mode 100644 news/12389.bugfix.rst create mode 100644 tools/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 999bd8b1d..18d911256 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,20 +28,20 @@ repos: args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.961 + rev: v1.6.1 hooks: - id: mypy exclude: tests/data args: ["--pretty", "--show-error-codes"] additional_dependencies: [ - 'keyring==23.0.1', - 'nox==2021.6.12', + 'keyring==24.2.0', + 'nox==2023.4.22', 'pytest', - 'types-docutils==0.18.3', - 'types-setuptools==57.4.14', - 'types-freezegun==1.1.9', - 'types-six==1.16.15', - 'types-pyyaml==6.0.12.2', + 'types-docutils==0.20.0.3', + 'types-setuptools==68.2.0.0', + 'types-freezegun==1.1.10', + 'types-six==1.16.21.9', + 'types-pyyaml==6.0.12.12', ] - repo: https://github.com/pre-commit/pygrep-hooks diff --git a/news/12389.bugfix.rst b/news/12389.bugfix.rst new file mode 100644 index 000000000..848718733 --- /dev/null +++ b/news/12389.bugfix.rst @@ -0,0 +1 @@ +Update mypy to 1.6.1 and fix/ignore types diff --git a/src/pip/_internal/locations/_distutils.py b/src/pip/_internal/locations/_distutils.py index 48689f5fb..0e18c6e1e 100644 --- a/src/pip/_internal/locations/_distutils.py +++ b/src/pip/_internal/locations/_distutils.py @@ -56,8 +56,7 @@ def distutils_scheme( try: d.parse_config_files() except UnicodeDecodeError: - # Typeshed does not include find_config_files() for some reason. - paths = d.find_config_files() # type: ignore + paths = d.find_config_files() logger.warning( "Ignore distutils configs in %s due to encoding errors.", ", ".join(os.path.basename(p) for p in paths), diff --git a/src/pip/_internal/metadata/_json.py b/src/pip/_internal/metadata/_json.py index 336b52f1e..27362fc72 100644 --- a/src/pip/_internal/metadata/_json.py +++ b/src/pip/_internal/metadata/_json.py @@ -64,10 +64,10 @@ def msg_to_json(msg: Message) -> Dict[str, Any]: key = json_name(field) if multi: value: Union[str, List[str]] = [ - sanitise_header(v) for v in msg.get_all(field) + sanitise_header(v) for v in msg.get_all(field) # type: ignore ] else: - value = sanitise_header(msg.get(field)) + value = sanitise_header(msg.get(field)) # type: ignore if key == "keywords": # Accept both comma-separated and space-separated # forms, for better compatibility with old data. diff --git a/src/pip/_internal/network/xmlrpc.py b/src/pip/_internal/network/xmlrpc.py index 4a7d55d0e..22ec8d2f4 100644 --- a/src/pip/_internal/network/xmlrpc.py +++ b/src/pip/_internal/network/xmlrpc.py @@ -13,6 +13,8 @@ from pip._internal.network.utils import raise_for_status if TYPE_CHECKING: from xmlrpc.client import _HostType, _Marshallable + from _typeshed import SizedBuffer + logger = logging.getLogger(__name__) @@ -33,7 +35,7 @@ class PipXmlrpcTransport(xmlrpc.client.Transport): self, host: "_HostType", handler: str, - request_body: bytes, + request_body: "SizedBuffer", verbose: bool = False, ) -> Tuple["_Marshallable", ...]: assert isinstance(host, str) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 42a8536cd..1ad3f6162 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -141,9 +141,9 @@ def rmtree( ) if sys.version_info >= (3, 12): # See https://docs.python.org/3.12/whatsnew/3.12.html#shutil. - shutil.rmtree(dir, onexc=handler) + shutil.rmtree(dir, onexc=handler) # type: ignore else: - shutil.rmtree(dir, onerror=handler) + shutil.rmtree(dir, onerror=handler) # type: ignore def _onerror_ignore(*_args: Any) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index 8e498abd0..c5bf4bb95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ from hashlib import sha256 from pathlib import Path from textwrap import dedent from typing import ( + TYPE_CHECKING, Any, AnyStr, Callable, @@ -58,6 +59,9 @@ from tests.lib import ( from tests.lib.server import MockServer, make_mock_server from tests.lib.venv import VirtualEnvironment, VirtualEnvironmentType +if TYPE_CHECKING: + from pip._vendor.typing_extensions import Self + def pytest_addoption(parser: Parser) -> None: parser.addoption( @@ -941,7 +945,7 @@ def html_index_with_onetime_server( """ class InDirectoryServer(http.server.ThreadingHTTPServer): - def finish_request(self, request: Any, client_address: Any) -> None: + def finish_request(self: "Self", request: Any, client_address: Any) -> None: self.RequestHandlerClass( request, client_address, diff --git a/tests/lib/configuration_helpers.py b/tests/lib/configuration_helpers.py index ec824ffd3..b6e398c5b 100644 --- a/tests/lib/configuration_helpers.py +++ b/tests/lib/configuration_helpers.py @@ -38,7 +38,7 @@ class ConfigurationMixin: old() # https://github.com/python/mypy/issues/2427 - self.configuration._load_config_files = overridden # type: ignore[assignment] + self.configuration._load_config_files = overridden # type: ignore[method-assign] @contextlib.contextmanager def tmpfile(self, contents: str) -> Iterator[str]: diff --git a/tests/lib/test_wheel.py b/tests/lib/test_wheel.py index 86994c28e..ffe96cc43 100644 --- a/tests/lib/test_wheel.py +++ b/tests/lib/test_wheel.py @@ -19,12 +19,12 @@ from tests.lib.wheel import ( def test_message_from_dict_one_value() -> None: message = message_from_dict({"a": "1"}) - assert set(message.get_all("a")) == {"1"} + assert set(message.get_all("a")) == {"1"} # type: ignore def test_message_from_dict_multiple_values() -> None: message = message_from_dict({"a": ["1", "2"]}) - assert set(message.get_all("a")) == {"1", "2"} + assert set(message.get_all("a")) == {"1", "2"} # type: ignore def message_from_bytes(contents: bytes) -> Message: @@ -67,7 +67,7 @@ def test_make_metadata_file_custom_value_list() -> None: f = default_make_metadata(updates={"a": ["1", "2"]}) assert f is not None message = default_metadata_checks(f) - assert set(message.get_all("a")) == {"1", "2"} + assert set(message.get_all("a")) == {"1", "2"} # type: ignore def test_make_metadata_file_custom_value_overrides() -> None: @@ -101,7 +101,7 @@ def default_wheel_metadata_checks(f: File) -> Message: assert message.get_all("Wheel-Version") == ["1.0"] assert message.get_all("Generator") == ["pip-test-suite"] assert message.get_all("Root-Is-Purelib") == ["true"] - assert set(message.get_all("Tag")) == {"py2-none-any", "py3-none-any"} + assert set(message.get_all("Tag")) == {"py2-none-any", "py3-none-any"} # type: ignore return message @@ -122,7 +122,7 @@ def test_make_wheel_metadata_file_custom_value_list() -> None: f = default_make_wheel_metadata(updates={"a": ["1", "2"]}) assert f is not None message = default_wheel_metadata_checks(f) - assert set(message.get_all("a")) == {"1", "2"} + assert set(message.get_all("a")) == {"1", "2"} # type: ignore def test_make_wheel_metadata_file_custom_value_override() -> None: diff --git a/tests/unit/metadata/test_metadata.py b/tests/unit/metadata/test_metadata.py index 47093fb54..ccc8ceb2e 100644 --- a/tests/unit/metadata/test_metadata.py +++ b/tests/unit/metadata/test_metadata.py @@ -23,7 +23,7 @@ def test_dist_get_direct_url_no_metadata(mock_read_text: mock.Mock) -> None: class FakeDistribution(BaseDistribution): pass - dist = FakeDistribution() + dist = FakeDistribution() # type: ignore assert dist.direct_url is None mock_read_text.assert_called_once_with(DIRECT_URL_METADATA_NAME) @@ -35,7 +35,7 @@ def test_dist_get_direct_url_invalid_json( class FakeDistribution(BaseDistribution): canonical_name = cast(NormalizedName, "whatever") # Needed for error logging. - dist = FakeDistribution() + dist = FakeDistribution() # type: ignore with caplog.at_level(logging.WARNING): assert dist.direct_url is None @@ -84,7 +84,7 @@ def test_dist_get_direct_url_valid_metadata(mock_read_text: mock.Mock) -> None: class FakeDistribution(BaseDistribution): pass - dist = FakeDistribution() + dist = FakeDistribution() # type: ignore direct_url = dist.direct_url assert direct_url is not None mock_read_text.assert_called_once_with(DIRECT_URL_METADATA_NAME) diff --git a/tests/unit/test_base_command.py b/tests/unit/test_base_command.py index daec5fc6c..44dae384a 100644 --- a/tests/unit/test_base_command.py +++ b/tests/unit/test_base_command.py @@ -151,7 +151,7 @@ def test_base_command_provides_tempdir_helpers() -> None: c = Command("fake", "fake") # https://github.com/python/mypy/issues/2427 - c.run = Mock(side_effect=assert_helpers_set) # type: ignore[assignment] + c.run = Mock(side_effect=assert_helpers_set) # type: ignore[method-assign] assert c.main(["fake"]) == SUCCESS c.run.assert_called_once() @@ -176,7 +176,7 @@ def test_base_command_global_tempdir_cleanup(kind: str, exists: bool) -> None: c = Command("fake", "fake") # https://github.com/python/mypy/issues/2427 - c.run = Mock(side_effect=create_temp_dirs) # type: ignore[assignment] + c.run = Mock(side_effect=create_temp_dirs) # type: ignore[method-assign] assert c.main(["fake"]) == SUCCESS c.run.assert_called_once() assert os.path.exists(Holder.value) == exists @@ -200,6 +200,6 @@ def test_base_command_local_tempdir_cleanup(kind: str, exists: bool) -> None: c = Command("fake", "fake") # https://github.com/python/mypy/issues/2427 - c.run = Mock(side_effect=create_temp_dirs) # type: ignore[assignment] + c.run = Mock(side_effect=create_temp_dirs) # type: ignore[method-assign] assert c.main(["fake"]) == SUCCESS c.run.assert_called_once() diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index c6b44d45a..1a0acb7b4 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -215,7 +215,7 @@ class TestConfigurationModification(ConfigurationMixin): # Mock out the method mymock = MagicMock(spec=self.configuration._mark_as_modified) # https://github.com/python/mypy/issues/2427 - self.configuration._mark_as_modified = mymock # type: ignore[assignment] + self.configuration._mark_as_modified = mymock # type: ignore[method-assign] self.configuration.set_value("test.hello", "10") @@ -231,7 +231,7 @@ class TestConfigurationModification(ConfigurationMixin): # Mock out the method mymock = MagicMock(spec=self.configuration._mark_as_modified) # https://github.com/python/mypy/issues/2427 - self.configuration._mark_as_modified = mymock # type: ignore[assignment] + self.configuration._mark_as_modified = mymock # type: ignore[method-assign] self.configuration.set_value("test.hello", "10") @@ -250,7 +250,7 @@ class TestConfigurationModification(ConfigurationMixin): # Mock out the method mymock = MagicMock(spec=self.configuration._mark_as_modified) # https://github.com/python/mypy/issues/2427 - self.configuration._mark_as_modified = mymock # type: ignore[assignment] + self.configuration._mark_as_modified = mymock # type: ignore[method-assign] self.configuration.set_value("test.hello", "10") diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 133e59222..b2f93b3d4 100644 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -252,7 +252,7 @@ class TestCheckDistRequiresPython: def metadata(self) -> email.message.Message: raise FileNotFoundError(metadata_name) - dist = make_fake_dist(klass=NotWorkingFakeDist) + dist = make_fake_dist(klass=NotWorkingFakeDist) # type: ignore with pytest.raises(NoneMetadataError) as exc: _check_dist_requires_python( diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 000000000..e69de29bb