From fdc262f06936fb406af2af74ad6b0946ac1f4bd8 Mon Sep 17 00:00:00 2001 From: Felipe Peter Date: Wed, 19 Oct 2022 14:23:06 +0800 Subject: [PATCH 001/216] Fix error message and improve help text --- news/3297dfd6-b078-4452-97a1-7d2c1ab41ca1.trivial.rst | 0 src/pip/_internal/commands/list.py | 9 +++++++-- tests/functional/test_list.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 news/3297dfd6-b078-4452-97a1-7d2c1ab41ca1.trivial.rst diff --git a/news/3297dfd6-b078-4452-97a1-7d2c1ab41ca1.trivial.rst b/news/3297dfd6-b078-4452-97a1-7d2c1ab41ca1.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 8e1426dbb..c4df1008a 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -103,7 +103,11 @@ class ListCommand(IndexGroupCommand): dest="list_format", default="columns", choices=("columns", "freeze", "json"), - help="Select the output format among: columns (default), freeze, or json", + help=( + "Select the output format among: columns (default), freeze, or json. " + "The 'freeze' format cannot be used together with the --outdated " + "option." + ), ) self.cmd_opts.add_option( @@ -157,7 +161,8 @@ class ListCommand(IndexGroupCommand): if options.outdated and options.list_format == "freeze": raise CommandError( - "List format 'freeze' can not be used with the --outdated option." + "List format 'freeze' cannot be used together with the --outdated " + "option." ) cmdoptions.check_list_path_option(options) diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index c7fdec2f2..cd94960b3 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -588,7 +588,7 @@ def test_outdated_formats(script: PipTestEnvironment, data: TestData) -> None: expect_error=True, ) assert ( - "List format 'freeze' can not be used with the --outdated option." + "List format 'freeze' cannot be used together with the --outdated option." in result.stderr ) From fb454572b2e455b22a59822b53b203b7232570e6 Mon Sep 17 00:00:00 2001 From: Wu Zhenyu Date: Sun, 28 Aug 2022 17:29:25 +0800 Subject: [PATCH 002/216] Fix `pip completion --zsh` --- news/11416.bugfix.rst | 1 + src/pip/_internal/commands/completion.py | 13 ++++--------- tests/functional/test_completion.py | 16 ++++++---------- 3 files changed, 11 insertions(+), 19 deletions(-) create mode 100644 news/11416.bugfix.rst diff --git a/news/11416.bugfix.rst b/news/11416.bugfix.rst new file mode 100644 index 000000000..3815b2da8 --- /dev/null +++ b/news/11416.bugfix.rst @@ -0,0 +1 @@ +Fix ``pip completion --zsh``. diff --git a/src/pip/_internal/commands/completion.py b/src/pip/_internal/commands/completion.py index deaa30899..30233fc7a 100644 --- a/src/pip/_internal/commands/completion.py +++ b/src/pip/_internal/commands/completion.py @@ -22,15 +22,10 @@ COMPLETION_SCRIPTS = { complete -o default -F _pip_completion {prog} """, "zsh": """ - function _pip_completion {{ - local words cword - read -Ac words - read -cn cword - reply=( $( COMP_WORDS="$words[*]" \\ - COMP_CWORD=$(( cword-1 )) \\ - PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )) - }} - compctl -K _pip_completion {prog} + #compdef -P pip[0-9.]# + compadd $( COMP_WORDS="$words[*]" \\ + COMP_CWORD=$((CURRENT-1)) \\ + PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null ) """, "fish": """ function __fish_complete_pip diff --git a/tests/functional/test_completion.py b/tests/functional/test_completion.py index b02cd4fa3..332dd1202 100644 --- a/tests/functional/test_completion.py +++ b/tests/functional/test_completion.py @@ -44,15 +44,10 @@ complete -fa "(__fish_complete_pip)" -c pip""", ( "zsh", """\ -function _pip_completion { - local words cword - read -Ac words - read -cn cword - reply=( $( COMP_WORDS="$words[*]" \\ - COMP_CWORD=$(( cword-1 )) \\ - PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )) -} -compctl -K _pip_completion pip""", +#compdef -P pip[0-9.]# +compadd $( COMP_WORDS="$words[*]" \\ + COMP_CWORD=$((CURRENT-1)) \\ + PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )""", ), ( "powershell", @@ -392,7 +387,8 @@ def test_completion_path_after_option( ) -@pytest.mark.parametrize("flag", ["--bash", "--zsh", "--fish", "--powershell"]) +# zsh completion script doesn't contain pip3 +@pytest.mark.parametrize("flag", ["--bash", "--fish", "--powershell"]) def test_completion_uses_same_executable_name( autocomplete_script: PipTestEnvironment, flag: str, deprecated_python: bool ) -> None: From 0621e5ad0d750f0fa7d06e5ec44a4bfd1e65e79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 10 Apr 2023 15:14:43 +0200 Subject: [PATCH 003/216] Add ireq.is_direct property, for readability --- src/pip/_internal/exceptions.py | 2 +- src/pip/_internal/models/installation_report.py | 2 +- src/pip/_internal/operations/prepare.py | 2 +- src/pip/_internal/req/req_install.py | 11 +++++++++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index 7d92ba699..d95fe44b3 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -544,7 +544,7 @@ class HashMissing(HashError): # so the output can be directly copied into the requirements file. package = ( self.req.original_link - if self.req.original_link + if self.req.is_direct # In case someone feeds something downright stupid # to InstallRequirement's constructor. else getattr(self.req, "req", None) diff --git a/src/pip/_internal/models/installation_report.py b/src/pip/_internal/models/installation_report.py index fef3757f2..7f001f35e 100644 --- a/src/pip/_internal/models/installation_report.py +++ b/src/pip/_internal/models/installation_report.py @@ -22,7 +22,7 @@ class InstallationReport: # is_direct is true if the requirement was a direct URL reference (which # includes editable requirements), and false if the requirement was # downloaded from a PEP 503 index or --find-links. - "is_direct": bool(ireq.original_link), + "is_direct": ireq.is_direct, # requested is true if the requirement was specified by the user (aka # top level requirement), and false if it was installed as a dependency of a # requirement. https://peps.python.org/pep-0376/#requested diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 227331523..dd265f255 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -352,7 +352,7 @@ class RequirementPreparer: # a surprising hash mismatch in the future. # file:/// URLs aren't pinnable, so don't complain about them # not being pinned. - if req.original_link is None and not req.is_pinned: + if not req.is_direct and not req.is_pinned: raise HashUnpinned() # If known-good hashes are missing for this requirement, diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index d01b24a91..1f479713a 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -104,6 +104,8 @@ class InstallRequirement: if link.is_file: self.source_dir = os.path.normpath(os.path.abspath(link.file_path)) + # original_link is the direct URL that was provided by the user for the + # requirement, either directly or via a constraints file. if link is None and req and req.url: # PEP 508 URL requirement link = Link(req.url) @@ -244,6 +246,11 @@ class InstallRequirement: def specifier(self) -> SpecifierSet: return self.req.specifier + @property + def is_direct(self) -> bool: + """Whether this requirement was specified as a direct URL.""" + return self.original_link is not None + @property def is_pinned(self) -> bool: """Return whether I am pinned to an exact version. @@ -293,7 +300,7 @@ class InstallRequirement: good_hashes = self.hash_options.copy() if trust_internet: link = self.link - elif self.original_link and self.user_supplied: + elif self.is_direct and self.user_supplied: link = self.original_link else: link = None @@ -804,7 +811,7 @@ class InstallRequirement: req_description=str(self.req), pycompile=pycompile, warn_script_location=warn_script_location, - direct_url=self.download_info if self.original_link else None, + direct_url=self.download_info if self.is_direct else None, requested=self.user_supplied, ) self.install_succeeded = True From 8295c9941b6516ccaaba9c91af025cb2e373dba0 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 24 Apr 2023 12:29:05 -0600 Subject: [PATCH 004/216] Deprecate .egg in the imporlib-metadata backend This provides us a path to remove all pkg_resources usages on Python 3.11 or later, and thus avoid the problem that pkg_resources uses Python API deprecated in 3.12. --- news/11996.process.rst | 1 + src/pip/_internal/metadata/importlib/_envs.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 news/11996.process.rst diff --git a/news/11996.process.rst b/news/11996.process.rst new file mode 100644 index 000000000..d585bd391 --- /dev/null +++ b/news/11996.process.rst @@ -0,0 +1 @@ +Deprecate support for eggs for Python 3.11 or later, when the new ``importlib.metadata`` backend is used to load distribution metadata. This only affects the egg *distribution format* (with the ``.egg`` extension); distributions using the ``.egg-info`` *metadata format* (but are not actually eggs) are not affected. For more information about eggs, see `relevant section in the setuptools documentation `__. diff --git a/src/pip/_internal/metadata/importlib/_envs.py b/src/pip/_internal/metadata/importlib/_envs.py index cbec59e2c..3850ddaf4 100644 --- a/src/pip/_internal/metadata/importlib/_envs.py +++ b/src/pip/_internal/metadata/importlib/_envs.py @@ -151,7 +151,7 @@ def _emit_egg_deprecation(location: Optional[str]) -> None: deprecated( reason=f"Loading egg at {location} is deprecated.", replacement="to use pip for package installation.", - gone_in=None, + gone_in="23.3", ) @@ -174,7 +174,7 @@ class Environment(BaseEnvironment): for location in self._paths: yield from finder.find(location) for dist in finder.find_eggs(location): - # _emit_egg_deprecation(dist.location) # TODO: Enable this. + _emit_egg_deprecation(dist.location) yield dist # This must go last because that's how pkg_resources tie-breaks. yield from finder.find_linked(location) From 21857784d6d7a9139711cf77f77da925fe9189ee Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 26 Apr 2023 12:53:24 -0600 Subject: [PATCH 005/216] Implement PEP 685 extra normalization in resolver All extras from user input or dependant package metadata are properly normalized for comparison and resolution. This ensures requests for extras from a dependant can always correctly find the normalized extra in the dependency, even if the requested extra name is not normalized. Note that this still relies on the declaration of extra names in the dependency's package metadata to be properly normalized when the package is built, since correct comparison between an extra name's normalized and non-normalized forms requires change to the metadata parsing logic, which is only available in packaging 22.0 and up, which pip does not use at the moment. --- news/11649.bugfix.rst | 5 ++++ .../_internal/resolution/resolvelib/base.py | 2 +- .../resolution/resolvelib/candidates.py | 2 +- .../resolution/resolvelib/factory.py | 20 +++++++++------- .../resolution/resolvelib/requirements.py | 2 +- tests/functional/test_install_extras.py | 24 ++++++++++++++++++- 6 files changed, 42 insertions(+), 13 deletions(-) create mode 100644 news/11649.bugfix.rst diff --git a/news/11649.bugfix.rst b/news/11649.bugfix.rst new file mode 100644 index 000000000..65511711f --- /dev/null +++ b/news/11649.bugfix.rst @@ -0,0 +1,5 @@ +Normalize extras according to :pep:`685` from package metadata in the resolver +for comparison. This ensures extras are correctly compared and merged as long +as the package providing the extra(s) is built with values normalized according +to the standard. Note, however, that this *does not* solve cases where the +package itself contains unnormalized extra values in the metadata. diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index b206692a0..0275385db 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -12,7 +12,7 @@ CandidateLookup = Tuple[Optional["Candidate"], Optional[InstallRequirement]] CandidateVersion = Union[LegacyVersion, Version] -def format_name(project: str, extras: FrozenSet[str]) -> str: +def format_name(project: NormalizedName, extras: FrozenSet[NormalizedName]) -> str: if not extras: return project canonical_extras = sorted(canonicalize_name(e) for e in extras) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 31020e27a..48ef9a16d 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -423,7 +423,7 @@ class ExtrasCandidate(Candidate): def __init__( self, base: BaseCandidate, - extras: FrozenSet[str], + extras: FrozenSet[NormalizedName], ) -> None: self.base = base self.extras = extras diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 0331297b8..6d1ec3163 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -112,7 +112,7 @@ class Factory: self._editable_candidate_cache: Cache[EditableCandidate] = {} self._installed_candidate_cache: Dict[str, AlreadyInstalledCandidate] = {} self._extras_candidate_cache: Dict[ - Tuple[int, FrozenSet[str]], ExtrasCandidate + Tuple[int, FrozenSet[NormalizedName]], ExtrasCandidate ] = {} if not ignore_installed: @@ -138,7 +138,9 @@ class Factory: raise UnsupportedWheel(msg) def _make_extras_candidate( - self, base: BaseCandidate, extras: FrozenSet[str] + self, + base: BaseCandidate, + extras: FrozenSet[NormalizedName], ) -> ExtrasCandidate: cache_key = (id(base), extras) try: @@ -151,7 +153,7 @@ class Factory: def _make_candidate_from_dist( self, dist: BaseDistribution, - extras: FrozenSet[str], + extras: FrozenSet[NormalizedName], template: InstallRequirement, ) -> Candidate: try: @@ -166,7 +168,7 @@ class Factory: def _make_candidate_from_link( self, link: Link, - extras: FrozenSet[str], + extras: FrozenSet[NormalizedName], template: InstallRequirement, name: Optional[NormalizedName], version: Optional[CandidateVersion], @@ -244,12 +246,12 @@ class Factory: assert template.req, "Candidates found on index must be PEP 508" name = canonicalize_name(template.req.name) - extras: FrozenSet[str] = frozenset() + extras: FrozenSet[NormalizedName] = frozenset() for ireq in ireqs: assert ireq.req, "Candidates found on index must be PEP 508" specifier &= ireq.req.specifier hashes &= ireq.hashes(trust_internet=False) - extras |= frozenset(ireq.extras) + extras |= frozenset(canonicalize_name(e) for e in ireq.extras) def _get_installed_candidate() -> Optional[Candidate]: """Get the candidate for the currently-installed version.""" @@ -325,7 +327,7 @@ class Factory: def _iter_explicit_candidates_from_base( self, base_requirements: Iterable[Requirement], - extras: FrozenSet[str], + extras: FrozenSet[NormalizedName], ) -> Iterator[Candidate]: """Produce explicit candidates from the base given an extra-ed package. @@ -392,7 +394,7 @@ class Factory: explicit_candidates.update( self._iter_explicit_candidates_from_base( requirements.get(parsed_requirement.name, ()), - frozenset(parsed_requirement.extras), + frozenset(canonicalize_name(e) for e in parsed_requirement.extras), ), ) @@ -452,7 +454,7 @@ class Factory: self._fail_if_link_is_unsupported_wheel(ireq.link) cand = self._make_candidate_from_link( ireq.link, - extras=frozenset(ireq.extras), + extras=frozenset(canonicalize_name(e) for e in ireq.extras), template=ireq, name=canonicalize_name(ireq.name) if ireq.name else None, version=None, diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index 06addc0dd..7d244c693 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -43,7 +43,7 @@ class SpecifierRequirement(Requirement): def __init__(self, ireq: InstallRequirement) -> None: assert ireq.link is None, "This is a link, not a specifier" self._ireq = ireq - self._extras = frozenset(ireq.extras) + self._extras = frozenset(canonicalize_name(e) for e in ireq.extras) def __str__(self) -> str: return str(self._ireq.req) diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index c6cef00fa..6f2a6bf43 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -4,7 +4,12 @@ from os.path import join import pytest -from tests.lib import PipTestEnvironment, ResolverVariant, TestData +from tests.lib import ( + PipTestEnvironment, + ResolverVariant, + TestData, + create_basic_wheel_for_package, +) @pytest.mark.network @@ -223,3 +228,20 @@ def test_install_extra_merging( if not fails_on_legacy or resolver_variant == "2020-resolver": expected = f"Successfully installed pkga-0.1 simple-{simple_version}" assert expected in result.stdout + + +def test_install_extras(script: PipTestEnvironment) -> None: + create_basic_wheel_for_package(script, "a", "1", depends=["b", "dep[x-y]"]) + create_basic_wheel_for_package(script, "b", "1", depends=["dep[x_y]"]) + create_basic_wheel_for_package(script, "dep", "1", extras={"x-y": ["meh"]}) + create_basic_wheel_for_package(script, "meh", "1") + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "a", + ) + script.assert_installed(a="1", b="1", dep="1", meh="1") From b9066d4b00a2fa2cc6529ecb0b5920465e0fb812 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 11 May 2023 13:00:47 +0800 Subject: [PATCH 006/216] Add test cases for normalized weird extra --- tests/functional/test_install_extras.py | 33 ++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index 6f2a6bf43..21da9d50e 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -155,25 +155,50 @@ def test_install_fails_if_extra_at_end( assert "Extras after version" in result.stderr -def test_install_special_extra(script: PipTestEnvironment) -> None: +@pytest.mark.parametrize( + "specified_extra, requested_extra", + [ + ("Hop_hOp-hoP", "Hop_hOp-hoP"), + pytest.param( + "Hop_hOp-hoP", + "hop-hop-hop", + marks=pytest.mark.xfail( + reason=( + "matching a normalized extra request against an" + "unnormalized extra in metadata requires PEP 685 support " + "in packaging (see pypa/pip#11445)." + ), + ), + ), + ("hop-hop-hop", "Hop_hOp-hoP"), + ], +) +def test_install_special_extra( + script: PipTestEnvironment, + specified_extra: str, + requested_extra: str, +) -> None: # Check that uppercase letters and '-' are dealt with # make a dummy project pkga_path = script.scratch_path / "pkga" pkga_path.mkdir() pkga_path.joinpath("setup.py").write_text( textwrap.dedent( - """ + f""" from setuptools import setup setup(name='pkga', version='0.1', - extras_require={'Hop_hOp-hoP': ['missing_pkg']}, + extras_require={{'{specified_extra}': ['missing_pkg']}}, ) """ ) ) result = script.pip( - "install", "--no-index", f"{pkga_path}[Hop_hOp-hoP]", expect_error=True + "install", + "--no-index", + f"{pkga_path}[{requested_extra}]", + expect_error=True, ) assert ( "Could not find a version that satisfies the requirement missing_pkg" From d64190c5fbbf38cf40215ef7122f1b8c6847afc9 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 11 May 2023 14:32:41 +0800 Subject: [PATCH 007/216] Try to find dependencies from unnormalized extras When an unnormalized extra is requested, try to look up dependencies with both its raw and normalized forms, to maintain compatibility when an extra is both specified and requested in a non-standard form. --- .../resolution/resolvelib/candidates.py | 62 ++++++++++++++----- .../resolution/resolvelib/factory.py | 18 +++--- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 48ef9a16d..b737bffc9 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -423,10 +423,17 @@ class ExtrasCandidate(Candidate): def __init__( self, base: BaseCandidate, - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], ) -> None: self.base = base - self.extras = extras + self.extras = frozenset(canonicalize_name(e) for e in extras) + # If any extras are requested in their non-normalized forms, keep track + # of their raw values. This is needed when we look up dependencies + # since PEP 685 has not been implemented for marker-matching, and using + # the non-normalized extra for lookup ensures the user can select a + # non-normalized extra in a package with its non-normalized form. + # TODO: Remove this when packaging is upgraded to support PEP 685. + self._unnormalized_extras = extras.difference(self.extras) def __str__(self) -> str: name, rest = str(self.base).split(" ", 1) @@ -477,6 +484,44 @@ class ExtrasCandidate(Candidate): def source_link(self) -> Optional[Link]: return self.base.source_link + def _warn_invalid_extras( + self, + requested: FrozenSet[str], + provided: FrozenSet[str], + ) -> None: + """Emit warnings for invalid extras being requested. + + This emits a warning for each requested extra that is not in the + candidate's ``Provides-Extra`` list. + """ + invalid_extras_to_warn = requested.difference( + provided, + # If an extra is requested in an unnormalized form, skip warning + # about the normalized form being missing. + (canonicalize_name(e) for e in self._unnormalized_extras), + ) + if not invalid_extras_to_warn: + return + for extra in sorted(invalid_extras_to_warn): + logger.warning( + "%s %s does not provide the extra '%s'", + self.base.name, + self.version, + extra, + ) + + def _calculate_valid_requested_extras(self) -> FrozenSet[str]: + """Get a list of valid extras requested by this candidate. + + The user (or upstream dependant) may have specified extras that the + candidate doesn't support. Any unsupported extras are dropped, and each + cause a warning to be logged here. + """ + requested_extras = self.extras.union(self._unnormalized_extras) + provided_extras = frozenset(self.base.dist.iter_provided_extras()) + self._warn_invalid_extras(requested_extras, provided_extras) + return requested_extras.intersection(provided_extras) + def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]: factory = self.base._factory @@ -486,18 +531,7 @@ class ExtrasCandidate(Candidate): if not with_requires: return - # The user may have specified extras that the candidate doesn't - # support. We ignore any unsupported extras here. - valid_extras = self.extras.intersection(self.base.dist.iter_provided_extras()) - invalid_extras = self.extras.difference(self.base.dist.iter_provided_extras()) - for extra in sorted(invalid_extras): - logger.warning( - "%s %s does not provide the extra '%s'", - self.base.name, - self.version, - extra, - ) - + valid_extras = self._calculate_valid_requested_extras() for r in self.base.dist.iter_dependencies(valid_extras): requirement = factory.make_requirement_from_spec( str(r), self.base._ireq, valid_extras diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 6d1ec3163..ff916236c 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -140,9 +140,9 @@ class Factory: def _make_extras_candidate( self, base: BaseCandidate, - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], ) -> ExtrasCandidate: - cache_key = (id(base), extras) + cache_key = (id(base), frozenset(canonicalize_name(e) for e in extras)) try: candidate = self._extras_candidate_cache[cache_key] except KeyError: @@ -153,7 +153,7 @@ class Factory: def _make_candidate_from_dist( self, dist: BaseDistribution, - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], template: InstallRequirement, ) -> Candidate: try: @@ -168,7 +168,7 @@ class Factory: def _make_candidate_from_link( self, link: Link, - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], template: InstallRequirement, name: Optional[NormalizedName], version: Optional[CandidateVersion], @@ -246,12 +246,12 @@ class Factory: assert template.req, "Candidates found on index must be PEP 508" name = canonicalize_name(template.req.name) - extras: FrozenSet[NormalizedName] = frozenset() + extras: FrozenSet[str] = frozenset() for ireq in ireqs: assert ireq.req, "Candidates found on index must be PEP 508" specifier &= ireq.req.specifier hashes &= ireq.hashes(trust_internet=False) - extras |= frozenset(canonicalize_name(e) for e in ireq.extras) + extras |= frozenset(ireq.extras) def _get_installed_candidate() -> Optional[Candidate]: """Get the candidate for the currently-installed version.""" @@ -327,7 +327,7 @@ class Factory: def _iter_explicit_candidates_from_base( self, base_requirements: Iterable[Requirement], - extras: FrozenSet[NormalizedName], + extras: FrozenSet[str], ) -> Iterator[Candidate]: """Produce explicit candidates from the base given an extra-ed package. @@ -394,7 +394,7 @@ class Factory: explicit_candidates.update( self._iter_explicit_candidates_from_base( requirements.get(parsed_requirement.name, ()), - frozenset(canonicalize_name(e) for e in parsed_requirement.extras), + frozenset(parsed_requirement.extras), ), ) @@ -454,7 +454,7 @@ class Factory: self._fail_if_link_is_unsupported_wheel(ireq.link) cand = self._make_candidate_from_link( ireq.link, - extras=frozenset(canonicalize_name(e) for e in ireq.extras), + extras=frozenset(ireq.extras), template=ireq, name=canonicalize_name(ireq.name) if ireq.name else None, version=None, From 4aa6d88ddcccde3e0f189b447f0c8886ceebe008 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 11 May 2023 15:12:30 +0800 Subject: [PATCH 008/216] Remove extra normalization from format_name util Since this function now always take normalized names, additional normalization is no longer needed. --- src/pip/_internal/resolution/resolvelib/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index 0275385db..9c0ef5ca7 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -1,7 +1,7 @@ from typing import FrozenSet, Iterable, Optional, Tuple, Union from pip._vendor.packaging.specifiers import SpecifierSet -from pip._vendor.packaging.utils import NormalizedName, canonicalize_name +from pip._vendor.packaging.utils import NormalizedName from pip._vendor.packaging.version import LegacyVersion, Version from pip._internal.models.link import Link, links_equivalent @@ -15,8 +15,8 @@ CandidateVersion = Union[LegacyVersion, Version] def format_name(project: NormalizedName, extras: FrozenSet[NormalizedName]) -> str: if not extras: return project - canonical_extras = sorted(canonicalize_name(e) for e in extras) - return "{}[{}]".format(project, ",".join(canonical_extras)) + extras_expr = ",".join(sorted(extras)) + return f"{project}[{extras_expr}]" class Constraint: From 07874252f50c3d423d3fdac2cb07ea31952348af Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 20 May 2023 13:22:38 +0200 Subject: [PATCH 009/216] make rejection message for count 8 slightly different --- src/pip/_internal/resolution/resolvelib/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/resolution/resolvelib/reporter.py b/src/pip/_internal/resolution/resolvelib/reporter.py index 3c724238a..3be4bd17e 100644 --- a/src/pip/_internal/resolution/resolvelib/reporter.py +++ b/src/pip/_internal/resolution/resolvelib/reporter.py @@ -22,7 +22,7 @@ class PipReporter(BaseReporter): 8: ( "pip is looking at multiple versions of {package_name} to " "determine which version is compatible with other " - "requirements. This could take a while." + "requirements. This could take quite a while." ), 13: ( "This is taking longer than usual. You might need to provide " From 1552934c088269437ca183a18a571833e94c8c05 Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 20 May 2023 17:12:03 +0200 Subject: [PATCH 010/216] Update reporter.py --- src/pip/_internal/resolution/resolvelib/reporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/resolution/resolvelib/reporter.py b/src/pip/_internal/resolution/resolvelib/reporter.py index 3be4bd17e..12adeff7b 100644 --- a/src/pip/_internal/resolution/resolvelib/reporter.py +++ b/src/pip/_internal/resolution/resolvelib/reporter.py @@ -20,9 +20,9 @@ class PipReporter(BaseReporter): "requirements. This could take a while." ), 8: ( - "pip is looking at multiple versions of {package_name} to " + "pip is still looking at multiple versions of {package_name} to " "determine which version is compatible with other " - "requirements. This could take quite a while." + "requirements. This could take a while." ), 13: ( "This is taking longer than usual. You might need to provide " From 1d74e7abbdc9a9ef0226ae1c6c410962bbdf501b Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 20 May 2023 17:39:36 +0200 Subject: [PATCH 011/216] Create 12040.feature.rst --- news/12040.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12040.feature.rst diff --git a/news/12040.feature.rst b/news/12040.feature.rst new file mode 100644 index 000000000..beff856f9 --- /dev/null +++ b/news/12040.feature.rst @@ -0,0 +1 @@ +make rejection messages slightly different between 1 and 8, so the user can make the difference. From e8b4bae05ead0e3929002838f5d4d56a9eb9bbc9 Mon Sep 17 00:00:00 2001 From: Felipe Peter Date: Sun, 21 May 2023 00:29:06 +0800 Subject: [PATCH 012/216] Remove superfluous "together" --- src/pip/_internal/commands/list.py | 6 ++---- tests/functional/test_list.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index c4df1008a..ac1035319 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -105,8 +105,7 @@ class ListCommand(IndexGroupCommand): choices=("columns", "freeze", "json"), help=( "Select the output format among: columns (default), freeze, or json. " - "The 'freeze' format cannot be used together with the --outdated " - "option." + "The 'freeze' format cannot be used with the --outdated option." ), ) @@ -161,8 +160,7 @@ class ListCommand(IndexGroupCommand): if options.outdated and options.list_format == "freeze": raise CommandError( - "List format 'freeze' cannot be used together with the --outdated " - "option." + "List format 'freeze' cannot be used with the --outdated option." ) cmdoptions.check_list_path_option(options) diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index d930346d1..4f2be8387 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -588,7 +588,7 @@ def test_outdated_formats(script: PipTestEnvironment, data: TestData) -> None: expect_error=True, ) assert ( - "List format 'freeze' cannot be used together with the --outdated option." + "List format 'freeze' cannot be used with the --outdated option." in result.stderr ) From 743e6a31af0b99411b489cef0cd5371ae5c9ca98 Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Sat, 20 May 2023 20:01:16 -0500 Subject: [PATCH 013/216] Add test for metadata mismatch (passes on main already) --- tests/functional/test_download.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 31418ca8c..3190c4004 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -1266,6 +1266,8 @@ class Package: metadata: MetadataKind # This will override any dependencies specified in the actual dist's METADATA. requires_dist: Tuple[str, ...] = () + # This will override the Name specified in the actual dist's METADATA. + metadata_name: str = None def metadata_filename(self) -> str: """This is specified by PEP 658.""" @@ -1296,7 +1298,7 @@ class Package: return dedent( f"""\ Metadata-Version: 2.1 - Name: {self.name} + Name: {self.metadata_name or self.name} Version: {self.version} {self.requires_str()} """ @@ -1452,6 +1454,14 @@ _simple_packages: Dict[str, List[Package]] = { ), # This will raise an error when pip attempts to fetch the metadata file. Package("simple2", "2.0", "simple2-2.0.tar.gz", MetadataKind.NoFile), + # This has a METADATA file with a mismatched name. + Package( + "simple2", + "3.0", + "simple2-3.0.tar.gz", + MetadataKind.Sha256, + metadata_name="not-simple2", + ), ], "colander": [ # Ensure we can read the dependencies from a metadata file within a wheel @@ -1581,3 +1591,18 @@ def test_metadata_not_found( f"ERROR: 404 Client Error: FileNotFoundError for url:.*{expected_re}" ) assert pattern.search(result.stderr), (pattern, result.stderr) + + +def test_produces_error_for_mismatched_package_name_in_metadata( + download_generated_html_index: Callable[..., Tuple[TestPipResult, Path]], +) -> None: + """Verify that the package name from the metadata matches the requested package.""" + result, _ = download_generated_html_index( + _simple_packages, + ["simple2==3.0"], + allow_error=True, + ) + assert result.returncode != 0 + assert ( + "simple2-3.0.tar.gz has inconsistent Name: expected 'simple2', but metadata has 'not-simple2'" + ) in result.stdout From 5f0dd4c60e19f903fdc5c2565967c4b9c7ed9e9d Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Sat, 20 May 2023 20:10:53 -0500 Subject: [PATCH 014/216] Add regression test for canonicalized package names for PEP658 metadata --- tests/functional/test_download.py | 45 ++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 3190c4004..8da185c06 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -8,7 +8,7 @@ from enum import Enum from hashlib import sha256 from pathlib import Path from textwrap import dedent -from typing import Callable, Dict, List, Tuple +from typing import Callable, Dict, List, Optional, Tuple import pytest @@ -1267,7 +1267,7 @@ class Package: # This will override any dependencies specified in the actual dist's METADATA. requires_dist: Tuple[str, ...] = () # This will override the Name specified in the actual dist's METADATA. - metadata_name: str = None + metadata_name: Optional[str] = None def metadata_filename(self) -> str: """This is specified by PEP 658.""" @@ -1501,6 +1501,16 @@ _simple_packages: Dict[str, List[Package]] = { "priority", "1.0", "priority-1.0-py2.py3-none-any.whl", MetadataKind.NoFile ), ], + "requires-simple-extra": [ + # Metadata name is not canonicalized. + Package( + "requires-simple-extra", + "0.1", + "requires_simple_extra-0.1-py2.py3-none-any.whl", + MetadataKind.Sha256, + metadata_name="Requires_Simple.Extra", + ), + ], } @@ -1604,5 +1614,34 @@ def test_produces_error_for_mismatched_package_name_in_metadata( ) assert result.returncode != 0 assert ( - "simple2-3.0.tar.gz has inconsistent Name: expected 'simple2', but metadata has 'not-simple2'" + "simple2-3.0.tar.gz has inconsistent Name: expected 'simple2', but metadata " + "has 'not-simple2'" ) in result.stdout + + +@pytest.mark.parametrize( + "requirement", + ( + "requires-simple-extra==0.1", + "REQUIRES_SIMPLE-EXTRA==0.1", + "REQUIRES....simple-_-EXTRA==0.1", + ), +) +def test_canonicalizes_package_name_before_verifying_metadata( + download_generated_html_index: Callable[..., Tuple[TestPipResult, Path]], + requirement: str, +) -> None: + """Verify that the package name from the command line and the package's + METADATA are both canonicalized before comparison. + + Regression test for https://github.com/pypa/pip/issues/12038 + """ + result, download_dir = download_generated_html_index( + _simple_packages, + [requirement], + allow_error=True, + ) + assert result.returncode == 0 + assert os.listdir(download_dir) == [ + "requires_simple_extra-0.1-py2.py3-none-any.whl", + ] From 767bb40755243e1519be262c44abbda81d06868b Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Sat, 20 May 2023 20:11:11 -0500 Subject: [PATCH 015/216] Canonicalize package names before comparison for PEP658 metadata --- news/12038.bugfix.rst | 1 + src/pip/_internal/operations/prepare.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 news/12038.bugfix.rst diff --git a/news/12038.bugfix.rst b/news/12038.bugfix.rst new file mode 100644 index 000000000..184d88dd8 --- /dev/null +++ b/news/12038.bugfix.rst @@ -0,0 +1 @@ +Fix installation of packages with PEP658 metadata using non-canonicalized names diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 227331523..6c41d26f5 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -410,7 +410,7 @@ class RequirementPreparer: # NB: raw_name will fall back to the name from the install requirement if # the Name: field is not present, but it's noted in the raw_name docstring # that that should NEVER happen anyway. - if metadata_dist.raw_name != req.req.name: + if canonicalize_name(metadata_dist.raw_name) != canonicalize_name(req.req.name): raise MetadataInconsistent( req, "Name", req.req.name, metadata_dist.raw_name ) From 3aaf5c32d6d069f1d3faa754dd4b7c7ea880037d Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 22 May 2023 00:40:20 +0100 Subject: [PATCH 016/216] Add `.git-blame-ignore-revs` for Black reformatting commits (#12045) Co-authored-by: Pradyun Gedam --- .git-blame-ignore-revs | 35 +++++++++++++++++++++++++++++++++++ MANIFEST.in | 1 + 2 files changed, 36 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000..c7644d0e6 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,35 @@ +917b41d6d73535c090fc312668dff353cdaef906 # Blacken docs/html/conf.py +ed383dd8afa8fe0250dcf9b8962927ada0e21c89 # Blacken docs/pip_sphinxext.py +228405e62451abe8a66233573035007df4be575f # Blacken noxfile.py +f477a9f490e978177b71c9dbaa5465c51ea21129 # Blacken setup.py +e59ba23468390217479465019f8d78e724a23550 # Blacken src/pip/__main__.py +d7013db084e9a52242354ee5754dc5d19ccf062e # Blacken src/pip/_internal/build_env.py +30e9ffacae75378fc3e3df48f754dabad037edb9 # Blacken src/pip/_internal/cache.py +8341d56b46776a805286218ac5fb0e7850fd9341 # Blacken src/pip/_internal/cli/autocompletion.py +3d3461ed65208656358b3595e25d8c31c5c89470 # Blacken src/pip/_internal/cli/base_command.py +d489b0f1b104bc936b0fb17e6c33633664ebdc0e # Blacken src/pip/_internal/cli/cmdoptions.py +591fe4841aefe9befa0530f2a54f820c4ecbb392 # Blacken src/pip/_internal/cli/command_context.py +9265b28ef7248ae1847a80384dbeeb8119c3e2f5 # Blacken src/pip/_internal/cli/main.py +847a369364878c38d210c90beed2737bb6fb3a85 # Blacken src/pip/_internal/cli/main_parser.py +ec97119067041ae58b963935ff5f0e5d9fead80c # Blacken src/pip/_internal/cli/parser.py +6e3b8de22fa39fa3073599ecf9db61367f4b3b32 # Blacken src/pip/_internal/cli/progress_bars.py +55405227de983c5bd5bf0858ea12dbe537d3e490 # Blacken src/pip/_internal/cli/req_command.py +d5ca5c850cae9a0c64882a8f49d3a318699a7e2e # Blacken src/pip/_internal/cli/spinners.py +9747cb48f8430a7a91b36fe697dd18dbddb319f0 # Blacken src/pip/_internal/commands/__init__.py +1c09fd6f124df08ca36bed68085ad68e89bb1957 # Blacken src/pip/_internal/commands/cache.py +315e93d7eb87cd476afcc4eaf0f01a7b56a5037f # Blacken src/pip/_internal/commands/check.py +8ae3b96ed7d24fd24024ccce4840da0dcf635f26 # Blacken src/pip/_internal/commands/completion.py +42ca4792202f26a293ee48380718743a80bbee37 # Blacken src/pip/_internal/commands/configuration.py +790ad78fcd43d41a5bef9dca34a3c128d05eb02c # Blacken src/pip/_internal/commands/debug.py +a6fcc8f045afe257ce321f4012fc8fcb4be01eb3 # Blacken src/pip/_internal/commands/download.py +920e735dfc60109351fbe2f4c483c2f6ede9e52d # Blacken src/pip/_internal/commands/freeze.py +053004e0fcf0851238b1064fbce13aea87b24e9c # Blacken src/pip/_internal/commands/hash.py +a6b6ae487e52c2242045b64cb8962e0a992cfd76 # Blacken src/pip/_internal/commands/help.py +2495cf95a6c7eb61ccf1f9f0e8b8d736af914e53 # Blacken __main__.py +c7ee560e00b85f7486b452c14ff49e4737996eda # Blacken tools/ +8e2e1964a4f0a060f7299a96a911c9e116b2283d # Blacken src/pip/_internal/commands/ +1bc0eef05679e87f45540ab0a294667cb3c6a88e # Blacken src/pip/_internal/network/ +069b01932a7d64a81c708c6254cc93e1f89e6783 # Blacken src/pip/_internal/req +1897784d59e0d5fcda2dd75fea54ddd8be3d502a # Blacken src/pip/_internal/index +94999255d5ede440c37137d210666fdf64302e75 # Reformat the codebase, with black +585037a80a1177f1fa92e159a7079855782e543e # Cleanup implicit string concatenation diff --git a/MANIFEST.in b/MANIFEST.in index 4716f4157..f896c0258 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,6 +14,7 @@ recursive-include src/pip/_vendor *COPYING* include docs/docutils.conf include docs/requirements.txt +exclude .git-blame-ignore-revs exclude .coveragerc exclude .mailmap exclude .appveyor.yml From 7de6455e11f1c2d149fe888654c547ebacf879c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 22 May 2023 13:27:06 +0200 Subject: [PATCH 017/216] Remove RIGHT-TO-LEFT OVERRIDE from AUTHORS.txt The names don't seem to be right-to-left anyway. --- AUTHORS.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AUTHORS.txt b/AUTHORS.txt index e9d3c3891..f6ece21ce 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -473,7 +473,7 @@ Miro Hrončok Monica Baluna montefra Monty Taylor -Muha Ajjan‮ +Muha Ajjan Nadav Wexler Nahuel Ambrosini Nate Coraor @@ -725,4 +725,4 @@ Zvezdan Petkovic Łukasz Langa Роман Донченко Семён Марьясин -‮rekcäH nitraM‮ +Martin Häcker From 75f54cae9271179b8cc80435f92336c97e349f9d Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 22 May 2023 17:20:03 -0600 Subject: [PATCH 018/216] Upgrade vendored Requests --- news/requests.vendor.rst | 1 + src/pip/_vendor/requests/__init__.py | 8 +-- src/pip/_vendor/requests/__version__.py | 4 +- src/pip/_vendor/requests/_internal_utils.py | 6 +- src/pip/_vendor/requests/adapters.py | 72 ++++----------------- src/pip/_vendor/requests/api.py | 6 +- src/pip/_vendor/requests/sessions.py | 4 +- src/pip/_vendor/requests/utils.py | 32 +++++---- src/pip/_vendor/vendor.txt | 2 +- 9 files changed, 51 insertions(+), 84 deletions(-) create mode 100644 news/requests.vendor.rst diff --git a/news/requests.vendor.rst b/news/requests.vendor.rst new file mode 100644 index 000000000..cf10d8cbb --- /dev/null +++ b/news/requests.vendor.rst @@ -0,0 +1 @@ +Upgrade Requests to 2.31.0 diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py index a47762480..10ff67ff4 100644 --- a/src/pip/_vendor/requests/__init__.py +++ b/src/pip/_vendor/requests/__init__.py @@ -63,10 +63,10 @@ def check_compatibility(urllib3_version, chardet_version, charset_normalizer_ver # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.26 - assert major == 1 - assert minor >= 21 - assert minor <= 26 + # urllib3 >= 1.21.1 + assert major >= 1 + if major == 1: + assert minor >= 21 # Check charset_normalizer for compatibility. if chardet_version: diff --git a/src/pip/_vendor/requests/__version__.py b/src/pip/_vendor/requests/__version__.py index 69be3dec7..5063c3f8e 100644 --- a/src/pip/_vendor/requests/__version__.py +++ b/src/pip/_vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = "requests" __description__ = "Python HTTP for Humans." __url__ = "https://requests.readthedocs.io" -__version__ = "2.28.2" -__build__ = 0x022802 +__version__ = "2.31.0" +__build__ = 0x023100 __author__ = "Kenneth Reitz" __author_email__ = "me@kennethreitz.org" __license__ = "Apache 2.0" diff --git a/src/pip/_vendor/requests/_internal_utils.py b/src/pip/_vendor/requests/_internal_utils.py index 7dc9bc533..f2cf635e2 100644 --- a/src/pip/_vendor/requests/_internal_utils.py +++ b/src/pip/_vendor/requests/_internal_utils.py @@ -14,9 +14,11 @@ _VALID_HEADER_NAME_RE_STR = re.compile(r"^[^:\s][^:\r\n]*$") _VALID_HEADER_VALUE_RE_BYTE = re.compile(rb"^\S[^\r\n]*$|^$") _VALID_HEADER_VALUE_RE_STR = re.compile(r"^\S[^\r\n]*$|^$") +_HEADER_VALIDATORS_STR = (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR) +_HEADER_VALIDATORS_BYTE = (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE) HEADER_VALIDATORS = { - bytes: (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE), - str: (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR), + bytes: _HEADER_VALIDATORS_BYTE, + str: _HEADER_VALIDATORS_STR, } diff --git a/src/pip/_vendor/requests/adapters.py b/src/pip/_vendor/requests/adapters.py index f68f7d467..10c176790 100644 --- a/src/pip/_vendor/requests/adapters.py +++ b/src/pip/_vendor/requests/adapters.py @@ -22,7 +22,6 @@ from pip._vendor.urllib3.exceptions import ProxyError as _ProxyError from pip._vendor.urllib3.exceptions import ReadTimeoutError, ResponseError from pip._vendor.urllib3.exceptions import SSLError as _SSLError from pip._vendor.urllib3.poolmanager import PoolManager, proxy_from_url -from pip._vendor.urllib3.response import HTTPResponse from pip._vendor.urllib3.util import Timeout as TimeoutSauce from pip._vendor.urllib3.util import parse_url from pip._vendor.urllib3.util.retry import Retry @@ -194,7 +193,6 @@ class HTTPAdapter(BaseAdapter): num_pools=connections, maxsize=maxsize, block=block, - strict=True, **pool_kwargs, ) @@ -485,63 +483,19 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=timeout, read=timeout) try: - if not chunked: - resp = conn.urlopen( - method=request.method, - url=url, - body=request.body, - headers=request.headers, - redirect=False, - assert_same_host=False, - preload_content=False, - decode_content=False, - retries=self.max_retries, - timeout=timeout, - ) - - # Send the request. - else: - if hasattr(conn, "proxy_pool"): - conn = conn.proxy_pool - - low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) - - try: - skip_host = "Host" in request.headers - low_conn.putrequest( - request.method, - url, - skip_accept_encoding=True, - skip_host=skip_host, - ) - - for header, value in request.headers.items(): - low_conn.putheader(header, value) - - low_conn.endheaders() - - for i in request.body: - low_conn.send(hex(len(i))[2:].encode("utf-8")) - low_conn.send(b"\r\n") - low_conn.send(i) - low_conn.send(b"\r\n") - low_conn.send(b"0\r\n\r\n") - - # Receive the response from the server - r = low_conn.getresponse() - - resp = HTTPResponse.from_httplib( - r, - pool=conn, - connection=low_conn, - preload_content=False, - decode_content=False, - ) - except Exception: - # If we hit any problems here, clean up the connection. - # Then, raise so that we can handle the actual exception. - low_conn.close() - raise + resp = conn.urlopen( + method=request.method, + url=url, + body=request.body, + headers=request.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False, + retries=self.max_retries, + timeout=timeout, + chunked=chunked, + ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) diff --git a/src/pip/_vendor/requests/api.py b/src/pip/_vendor/requests/api.py index 2f71aaed1..cd0b3eeac 100644 --- a/src/pip/_vendor/requests/api.py +++ b/src/pip/_vendor/requests/api.py @@ -106,7 +106,7 @@ def post(url, data=None, json=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. + :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -121,7 +121,7 @@ def put(url, data=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. + :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -136,7 +136,7 @@ def patch(url, data=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. + :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response diff --git a/src/pip/_vendor/requests/sessions.py b/src/pip/_vendor/requests/sessions.py index 6cb3b4dae..dbcf2a7b0 100644 --- a/src/pip/_vendor/requests/sessions.py +++ b/src/pip/_vendor/requests/sessions.py @@ -324,7 +324,9 @@ class SessionRedirectMixin: except KeyError: username, password = None, None - if username and password: + # urllib3 handles proxy authorization for us in the standard adapter. + # Avoid appending this to TLS tunneled requests where it may be leaked. + if not scheme.startswith('https') and username and password: headers["Proxy-Authorization"] = _basic_auth_str(username, password) return new_proxies diff --git a/src/pip/_vendor/requests/utils.py b/src/pip/_vendor/requests/utils.py index 33f394d26..36607eda2 100644 --- a/src/pip/_vendor/requests/utils.py +++ b/src/pip/_vendor/requests/utils.py @@ -25,7 +25,12 @@ from . import certs from .__version__ import __version__ # to_native_string is unused here, but imported here for backwards compatibility -from ._internal_utils import HEADER_VALIDATORS, to_native_string # noqa: F401 +from ._internal_utils import ( # noqa: F401 + _HEADER_VALIDATORS_BYTE, + _HEADER_VALIDATORS_STR, + HEADER_VALIDATORS, + to_native_string, +) from .compat import ( Mapping, basestring, @@ -1031,20 +1036,23 @@ def check_header_validity(header): :param header: tuple, in the format (name, value). """ name, value = header - - for part in header: - if type(part) not in HEADER_VALIDATORS: - raise InvalidHeader( - f"Header part ({part!r}) from {{{name!r}: {value!r}}} must be " - f"of type str or bytes, not {type(part)}" - ) - - _validate_header_part(name, "name", HEADER_VALIDATORS[type(name)][0]) - _validate_header_part(value, "value", HEADER_VALIDATORS[type(value)][1]) + _validate_header_part(header, name, 0) + _validate_header_part(header, value, 1) -def _validate_header_part(header_part, header_kind, validator): +def _validate_header_part(header, header_part, header_validator_index): + if isinstance(header_part, str): + validator = _HEADER_VALIDATORS_STR[header_validator_index] + elif isinstance(header_part, bytes): + validator = _HEADER_VALIDATORS_BYTE[header_validator_index] + else: + raise InvalidHeader( + f"Header part ({header_part!r}) from {header} " + f"must be of type str or bytes, not {type(header_part)}" + ) + if not validator.match(header_part): + header_kind = "name" if header_validator_index == 0 else "value" raise InvalidHeader( f"Invalid leading whitespace, reserved character(s), or return" f"character(s) in header {header_kind}: {header_part!r}" diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 61063459d..732e90217 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -7,7 +7,7 @@ packaging==21.3 platformdirs==3.2.0 pyparsing==3.0.9 pyproject-hooks==1.0.0 -requests==2.28.2 +requests==2.31.0 certifi==2022.12.7 chardet==5.1.0 idna==3.4 From 1ca4529dc02902830be2a668a4e598f22f154071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 29 May 2023 12:34:21 +0200 Subject: [PATCH 019/216] Ignore invalid origin.json in wheel cache --- news/11985.bugfix.rst | 1 + src/pip/_internal/cache.py | 42 ++++++++++++++++++++++++++++---------- tests/unit/test_req.py | 19 +++++++++++++++++ 3 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 news/11985.bugfix.rst diff --git a/news/11985.bugfix.rst b/news/11985.bugfix.rst new file mode 100644 index 000000000..66c8e8786 --- /dev/null +++ b/news/11985.bugfix.rst @@ -0,0 +1 @@ +Ignore invalid or unreadable ``origin.json`` files in the cache of locally built wheels. diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 05f0a9acb..8d3a664c7 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -194,7 +194,17 @@ class CacheEntry: self.origin: Optional[DirectUrl] = None origin_direct_url_path = Path(self.link.file_path).parent / ORIGIN_JSON_NAME if origin_direct_url_path.exists(): - self.origin = DirectUrl.from_json(origin_direct_url_path.read_text()) + try: + self.origin = DirectUrl.from_json( + origin_direct_url_path.read_text(encoding="utf-8") + ) + except Exception as e: + logger.warning( + "Ignoring invalid cache entry origin file %s for %s (%s)", + origin_direct_url_path, + link.filename, + e, + ) class WheelCache(Cache): @@ -257,16 +267,26 @@ class WheelCache(Cache): @staticmethod def record_download_origin(cache_dir: str, download_info: DirectUrl) -> None: origin_path = Path(cache_dir) / ORIGIN_JSON_NAME - if origin_path.is_file(): - origin = DirectUrl.from_json(origin_path.read_text()) - # TODO: use DirectUrl.equivalent when https://github.com/pypa/pip/pull/10564 - # is merged. - if origin.url != download_info.url: + if origin_path.exists(): + try: + origin = DirectUrl.from_json(origin_path.read_text(encoding="utf-8")) + except Exception as e: logger.warning( - "Origin URL %s in cache entry %s does not match download URL %s. " - "This is likely a pip bug or a cache corruption issue.", - origin.url, - cache_dir, - download_info.url, + "Could not read origin file %s in cache entry (%s). " + "Will attempt to overwrite it.", + origin_path, + e, ) + else: + # TODO: use DirectUrl.equivalent when + # https://github.com/pypa/pip/pull/10564 is merged. + if origin.url != download_info.url: + logger.warning( + "Origin URL %s in cache entry %s does not match download URL " + "%s. This is likely a pip bug or a cache corruption issue. " + "Will overwrite it with the new value.", + origin.url, + cache_dir, + download_info.url, + ) origin_path.write_text(download_info.to_json(), encoding="utf-8") diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index c9742812b..74b9712dc 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -445,6 +445,25 @@ class TestRequirementSet: assert isinstance(req.download_info.info, ArchiveInfo) assert req.download_info.info.hash == hash + def test_download_info_archive_cache_with_invalid_origin( + self, tmp_path: Path, shared_data: TestData, caplog: pytest.LogCaptureFixture + ) -> None: + """Test an invalid origin.json is ignored.""" + url = shared_data.packages.joinpath("simple-1.0.tar.gz").as_uri() + finder = make_test_finder() + wheel_cache = WheelCache(str(tmp_path / "cache")) + cache_entry_dir = wheel_cache.get_path_for_link(Link(url)) + Path(cache_entry_dir).mkdir(parents=True) + Path(cache_entry_dir).joinpath("origin.json").write_text("{") # invalid json + wheel.make_wheel(name="simple", version="1.0").save_to_dir(cache_entry_dir) + with self._basic_resolver(finder, wheel_cache=wheel_cache) as resolver: + ireq = get_processed_req_from_line(f"simple @ {url}") + reqset = resolver.resolve([ireq], True) + assert len(reqset.all_requirements) == 1 + req = reqset.all_requirements[0] + assert req.is_wheel_from_cache + assert "Ignoring invalid cache entry origin file" in caplog.messages[0] + def test_download_info_local_wheel(self, data: TestData) -> None: """Test that download_info is set for requirements from a local wheel.""" finder = make_test_finder() From ab1b312cc5c42efeec21eba71b00a12a706e0162 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 29 May 2023 09:51:08 -0500 Subject: [PATCH 020/216] Update vendored typing extensions to 4.6 (#12056) --- news/typing-extensions.vendor.rst | 1 + src/pip/_vendor/typing_extensions.py | 858 +++++++++++++++++++++------ src/pip/_vendor/vendor.txt | 2 +- 3 files changed, 682 insertions(+), 179 deletions(-) create mode 100644 news/typing-extensions.vendor.rst diff --git a/news/typing-extensions.vendor.rst b/news/typing-extensions.vendor.rst new file mode 100644 index 000000000..6c95fc1b8 --- /dev/null +++ b/news/typing-extensions.vendor.rst @@ -0,0 +1 @@ +Updated typing_extensions to 4.6.0 diff --git a/src/pip/_vendor/typing_extensions.py b/src/pip/_vendor/typing_extensions.py index 9cbf5b87b..ae740bff7 100644 --- a/src/pip/_vendor/typing_extensions.py +++ b/src/pip/_vendor/typing_extensions.py @@ -33,6 +33,7 @@ __all__ = [ 'Coroutine', 'AsyncGenerator', 'AsyncContextManager', + 'Buffer', 'ChainMap', # Concrete collection types. @@ -45,7 +46,13 @@ __all__ = [ 'TypedDict', # Structural checks, a.k.a. protocols. + 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', + 'SupportsFloat', 'SupportsIndex', + 'SupportsInt', + 'SupportsRound', # One-off things. 'Annotated', @@ -58,6 +65,7 @@ __all__ = [ 'final', 'get_args', 'get_origin', + 'get_original_bases', 'get_type_hints', 'IntVar', 'is_typeddict', @@ -71,6 +79,7 @@ __all__ = [ 'runtime_checkable', 'Text', 'TypeAlias', + 'TypeAliasType', 'TypeGuard', 'TYPE_CHECKING', 'Never', @@ -86,7 +95,13 @@ GenericMeta = type # The functions below are modified copies of typing internal helpers. # They are needed by _ProtocolMeta and they provide support for PEP 646. -_marker = object() + +class _Sentinel: + def __repr__(self): + return "" + + +_marker = _Sentinel() def _check_generic(cls, parameters, elen=_marker): @@ -260,21 +275,70 @@ def IntVar(name): return typing.TypeVar(name) -# 3.8+: -if hasattr(typing, 'Literal'): +# A Literal bug was fixed in 3.11.0, 3.10.1 and 3.9.8 +if sys.version_info >= (3, 10, 1): Literal = typing.Literal -# 3.7: else: + def _flatten_literal_params(parameters): + """An internal helper for Literal creation: flatten Literals among parameters""" + params = [] + for p in parameters: + if isinstance(p, _LiteralGenericAlias): + params.extend(p.__args__) + else: + params.append(p) + return tuple(params) + + def _value_and_type_iter(params): + for p in params: + yield p, type(p) + + class _LiteralGenericAlias(typing._GenericAlias, _root=True): + def __eq__(self, other): + if not isinstance(other, _LiteralGenericAlias): + return NotImplemented + these_args_deduped = set(_value_and_type_iter(self.__args__)) + other_args_deduped = set(_value_and_type_iter(other.__args__)) + return these_args_deduped == other_args_deduped + + def __hash__(self): + return hash(frozenset(_value_and_type_iter(self.__args__))) + class _LiteralForm(typing._SpecialForm, _root=True): + def __init__(self, doc: str): + self._name = 'Literal' + self._doc = self.__doc__ = doc def __repr__(self): return 'typing_extensions.' + self._name def __getitem__(self, parameters): - return typing._GenericAlias(self, parameters) + if not isinstance(parameters, tuple): + parameters = (parameters,) - Literal = _LiteralForm('Literal', - doc="""A type that can be used to indicate to type checkers + parameters = _flatten_literal_params(parameters) + + val_type_pairs = list(_value_and_type_iter(parameters)) + try: + deduped_pairs = set(val_type_pairs) + except TypeError: + # unhashable parameters + pass + else: + # similar logic to typing._deduplicate on Python 3.9+ + if len(deduped_pairs) < len(val_type_pairs): + new_parameters = [] + for pair in val_type_pairs: + if pair in deduped_pairs: + new_parameters.append(pair[0]) + deduped_pairs.remove(pair) + assert not deduped_pairs, deduped_pairs + parameters = tuple(new_parameters) + + return _LiteralGenericAlias(self, parameters) + + Literal = _LiteralForm(doc="""\ + A type that can be used to indicate to type checkers that the corresponding value has a value literally equivalent to the provided parameter. For example: @@ -288,7 +352,7 @@ else: instead of a type.""") -_overload_dummy = typing._overload_dummy # noqa +_overload_dummy = typing._overload_dummy if hasattr(typing, "get_overloads"): # 3.11+ @@ -383,40 +447,54 @@ else: Counter = typing.Counter ChainMap = typing.ChainMap AsyncGenerator = typing.AsyncGenerator -NewType = typing.NewType Text = typing.Text TYPE_CHECKING = typing.TYPE_CHECKING -_PROTO_WHITELIST = ['Callable', 'Awaitable', - 'Iterable', 'Iterator', 'AsyncIterable', 'AsyncIterator', - 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', - 'ContextManager', 'AsyncContextManager'] +_PROTO_ALLOWLIST = { + 'collections.abc': [ + 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', + 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', + ], + 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], +} + + +_EXCLUDED_ATTRS = { + "__abstractmethods__", "__annotations__", "__weakref__", "_is_protocol", + "_is_runtime_protocol", "__dict__", "__slots__", "__parameters__", + "__orig_bases__", "__module__", "_MutableMapping__marker", "__doc__", + "__subclasshook__", "__orig_class__", "__init__", "__new__", + "__protocol_attrs__", "__callable_proto_members_only__", +} + +if sys.version_info < (3, 8): + _EXCLUDED_ATTRS |= { + "_gorg", "__next_in_mro__", "__extra__", "__tree_hash__", "__args__", + "__origin__" + } + +if sys.version_info >= (3, 9): + _EXCLUDED_ATTRS.add("__class_getitem__") + +if sys.version_info >= (3, 12): + _EXCLUDED_ATTRS.add("__type_params__") + +_EXCLUDED_ATTRS = frozenset(_EXCLUDED_ATTRS) def _get_protocol_attrs(cls): attrs = set() for base in cls.__mro__[:-1]: # without object - if base.__name__ in ('Protocol', 'Generic'): + if base.__name__ in {'Protocol', 'Generic'}: continue annotations = getattr(base, '__annotations__', {}) - for attr in list(base.__dict__.keys()) + list(annotations.keys()): - if (not attr.startswith('_abc_') and attr not in ( - '__abstractmethods__', '__annotations__', '__weakref__', - '_is_protocol', '_is_runtime_protocol', '__dict__', - '__args__', '__slots__', - '__next_in_mro__', '__parameters__', '__origin__', - '__orig_bases__', '__extra__', '__tree_hash__', - '__doc__', '__subclasshook__', '__init__', '__new__', - '__module__', '_MutableMapping__marker', '_gorg')): + for attr in (*base.__dict__, *annotations): + if (not attr.startswith('_abc_') and attr not in _EXCLUDED_ATTRS): attrs.add(attr) return attrs -def _is_callable_members_only(cls): - return all(callable(getattr(cls, attr, None)) for attr in _get_protocol_attrs(cls)) - - def _maybe_adjust_parameters(cls): """Helper function used in Protocol.__init_subclass__ and _TypedDictMeta.__new__. @@ -426,7 +504,7 @@ def _maybe_adjust_parameters(cls): """ tvars = [] if '__orig_bases__' in cls.__dict__: - tvars = typing._collect_type_vars(cls.__orig_bases__) + tvars = _collect_type_vars(cls.__orig_bases__) # Look for Generic[T1, ..., Tn] or Protocol[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. @@ -457,33 +535,82 @@ def _maybe_adjust_parameters(cls): cls.__parameters__ = tuple(tvars) -# 3.8+ -if hasattr(typing, 'Protocol'): +def _caller(depth=2): + try: + return sys._getframe(depth).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): # For platforms without _getframe() + return None + + +# The performance of runtime-checkable protocols is significantly improved on Python 3.12, +# so we backport the 3.12 version of Protocol to Python <=3.11 +if sys.version_info >= (3, 12): Protocol = typing.Protocol -# 3.7 + runtime_checkable = typing.runtime_checkable else: + def _allow_reckless_class_checks(depth=4): + """Allow instance and class checks for special stdlib modules. + The abc and functools modules indiscriminately call isinstance() and + issubclass() on the whole MRO of a user class, which may contain protocols. + """ + return _caller(depth) in {'abc', 'functools', None} def _no_init(self, *args, **kwargs): if type(self)._is_protocol: raise TypeError('Protocols cannot be instantiated') - class _ProtocolMeta(abc.ABCMeta): # noqa: B024 - # This metaclass is a bit unfortunate and exists only because of the lack - # of __instancehook__. + class _ProtocolMeta(abc.ABCMeta): + # This metaclass is somewhat unfortunate, + # but is necessary for several reasons... + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + if getattr(cls, "_is_protocol", False): + cls.__protocol_attrs__ = _get_protocol_attrs(cls) + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + cls.__callable_proto_members_only__ = all( + callable(getattr(cls, attr, None)) for attr in cls.__protocol_attrs__ + ) + + def __subclasscheck__(cls, other): + if ( + getattr(cls, '_is_protocol', False) + and not cls.__callable_proto_members_only__ + and not _allow_reckless_class_checks(depth=3) + ): + raise TypeError( + "Protocols with non-method members don't support issubclass()" + ) + return super().__subclasscheck__(other) + def __instancecheck__(cls, instance): # We need this method for situations where attributes are # assigned in __init__. - if ((not getattr(cls, '_is_protocol', False) or - _is_callable_members_only(cls)) and - issubclass(instance.__class__, cls)): + if not getattr(cls, "_is_protocol", False): + # i.e., it's a concrete subclass of a protocol + return super().__instancecheck__(instance) + + if ( + not getattr(cls, '_is_runtime_protocol', False) and + not _allow_reckless_class_checks(depth=2) + ): + raise TypeError("Instance and class checks can only be used with" + " @runtime_checkable protocols") + + if super().__instancecheck__(instance): return True - if cls._is_protocol: - if all(hasattr(instance, attr) and - (not callable(getattr(cls, attr, None)) or - getattr(instance, attr) is not None) - for attr in _get_protocol_attrs(cls)): - return True - return super().__instancecheck__(instance) + + for attr in cls.__protocol_attrs__: + try: + val = inspect.getattr_static(instance, attr) + except AttributeError: + break + if val is None and callable(getattr(cls, attr, None)): + break + else: + return True + + return False class Protocol(metaclass=_ProtocolMeta): # There is quite a lot of overlapping code with typing.Generic. @@ -519,6 +646,7 @@ else: """ __slots__ = () _is_protocol = True + _is_runtime_protocol = False def __new__(cls, *args, **kwds): if cls is Protocol: @@ -534,7 +662,7 @@ else: raise TypeError( f"Parameter list to {cls.__qualname__}[...] cannot be empty") msg = "Parameters to generic types must be types." - params = tuple(typing._type_check(p, msg) for p in params) # noqa + params = tuple(typing._type_check(p, msg) for p in params) if cls is Protocol: # Generic can only be subscripted with unique type variables. if not all(isinstance(p, typing.TypeVar) for p in params): @@ -570,19 +698,14 @@ else: if not cls.__dict__.get('_is_protocol', None): return NotImplemented if not getattr(cls, '_is_runtime_protocol', False): - if sys._getframe(2).f_globals['__name__'] in ['abc', 'functools']: + if _allow_reckless_class_checks(): return NotImplemented raise TypeError("Instance and class checks can only be used with" " @runtime protocols") - if not _is_callable_members_only(cls): - if sys._getframe(2).f_globals['__name__'] in ['abc', 'functools']: - return NotImplemented - raise TypeError("Protocols with non-method members" - " don't support issubclass()") if not isinstance(other, type): # Same error as for issubclass(1, int) raise TypeError('issubclass() arg 1 must be a class') - for attr in _get_protocol_attrs(cls): + for attr in cls.__protocol_attrs__: for base in other.__mro__: if attr in base.__dict__: if base.__dict__[attr] is None: @@ -607,19 +730,14 @@ else: # Check consistency of bases. for base in cls.__bases__: if not (base in (object, typing.Generic) or - base.__module__ == 'collections.abc' and - base.__name__ in _PROTO_WHITELIST or + base.__module__ in _PROTO_ALLOWLIST and + base.__name__ in _PROTO_ALLOWLIST[base.__module__] or isinstance(base, _ProtocolMeta) and base._is_protocol): raise TypeError('Protocols can only inherit from other' f' protocols, got {repr(base)}') - cls.__init__ = _no_init + if cls.__init__ is Protocol.__init__: + cls.__init__ = _no_init - -# 3.8+ -if hasattr(typing, 'runtime_checkable'): - runtime_checkable = typing.runtime_checkable -# 3.7 -else: def runtime_checkable(cls): """Mark a protocol class as a runtime protocol, so that it can be used with isinstance() and issubclass(). Raise TypeError @@ -628,7 +746,10 @@ else: This allows a simple-minded structural check very similar to the one-offs in collections.abc such as Hashable. """ - if not isinstance(cls, _ProtocolMeta) or not cls._is_protocol: + if not ( + (isinstance(cls, _ProtocolMeta) or issubclass(cls, typing.Generic)) + and getattr(cls, "_is_protocol", False) + ): raise TypeError('@runtime_checkable can be only applied to protocol classes,' f' got {cls!r}') cls._is_runtime_protocol = True @@ -639,11 +760,52 @@ else: runtime = runtime_checkable -# 3.8+ -if hasattr(typing, 'SupportsIndex'): +# Our version of runtime-checkable protocols is faster on Python 3.7-3.11 +if sys.version_info >= (3, 12): + SupportsInt = typing.SupportsInt + SupportsFloat = typing.SupportsFloat + SupportsComplex = typing.SupportsComplex + SupportsBytes = typing.SupportsBytes SupportsIndex = typing.SupportsIndex -# 3.7 + SupportsAbs = typing.SupportsAbs + SupportsRound = typing.SupportsRound else: + @runtime_checkable + class SupportsInt(Protocol): + """An ABC with one abstract method __int__.""" + __slots__ = () + + @abc.abstractmethod + def __int__(self) -> int: + pass + + @runtime_checkable + class SupportsFloat(Protocol): + """An ABC with one abstract method __float__.""" + __slots__ = () + + @abc.abstractmethod + def __float__(self) -> float: + pass + + @runtime_checkable + class SupportsComplex(Protocol): + """An ABC with one abstract method __complex__.""" + __slots__ = () + + @abc.abstractmethod + def __complex__(self) -> complex: + pass + + @runtime_checkable + class SupportsBytes(Protocol): + """An ABC with one abstract method __bytes__.""" + __slots__ = () + + @abc.abstractmethod + def __bytes__(self) -> bytes: + pass + @runtime_checkable class SupportsIndex(Protocol): __slots__ = () @@ -652,8 +814,30 @@ else: def __index__(self) -> int: pass + @runtime_checkable + class SupportsAbs(Protocol[T_co]): + """ + An ABC with one abstract method __abs__ that is covariant in its return type. + """ + __slots__ = () -if hasattr(typing, "Required"): + @abc.abstractmethod + def __abs__(self) -> T_co: + pass + + @runtime_checkable + class SupportsRound(Protocol[T_co]): + """ + An ABC with one abstract method __round__ that is covariant in its return type. + """ + __slots__ = () + + @abc.abstractmethod + def __round__(self, ndigits: int = 0) -> T_co: + pass + + +if sys.version_info >= (3, 12): # The standard library TypedDict in Python 3.8 does not store runtime information # about which (if any) keys are optional. See https://bugs.python.org/issue38834 # The standard library TypedDict in Python 3.9.0/1 does not honour the "total" @@ -661,15 +845,15 @@ if hasattr(typing, "Required"): # The standard library TypedDict below Python 3.11 does not store runtime # information about optional and required keys when using Required or NotRequired. # Generic TypedDicts are also impossible using typing.TypedDict on Python <3.11. + # Aaaand on 3.12 we add __orig_bases__ to TypedDict + # to enable better runtime introspection. TypedDict = typing.TypedDict _TypedDictMeta = typing._TypedDictMeta is_typeddict = typing.is_typeddict else: def _check_fails(cls, other): try: - if sys._getframe(1).f_globals['__name__'] not in ['abc', - 'functools', - 'typing']: + if _caller() not in {'abc', 'functools', 'typing'}: # Typed dicts are only for static structural subtyping. raise TypeError('TypedDict does not support instance and class checks') except (AttributeError, ValueError): @@ -692,7 +876,6 @@ else: typename, args = args[0], args[1:] # allow the "_typename" keyword be passed elif '_typename' in kwargs: typename = kwargs.pop('_typename') - import warnings warnings.warn("Passing '_typename' as keyword argument is deprecated", DeprecationWarning, stacklevel=2) else: @@ -707,7 +890,6 @@ else: 'were given') elif '_fields' in kwargs and len(kwargs) == 1: fields = kwargs.pop('_fields') - import warnings warnings.warn("Passing '_fields' as keyword argument is deprecated", DeprecationWarning, stacklevel=2) else: @@ -719,12 +901,20 @@ else: raise TypeError("TypedDict takes either a dict or keyword arguments," " but not both") + if kwargs: + warnings.warn( + "The kwargs-based syntax for TypedDict definitions is deprecated, " + "may be removed in a future version, and may not be " + "understood by third-party type checkers.", + DeprecationWarning, + stacklevel=2, + ) + ns = {'__annotations__': dict(fields)} - try: + module = _caller() + if module is not None: # Setting correct module is necessary to make typed dict classes pickleable. - ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass + ns['__module__'] = module return _TypedDictMeta(typename, (), ns, total=total) @@ -751,9 +941,14 @@ else: # Instead, monkey-patch __bases__ onto the class after it's been created. tp_dict = super().__new__(cls, name, (dict,), ns) - if any(issubclass(base, typing.Generic) for base in bases): + is_generic = any(issubclass(base, typing.Generic) for base in bases) + + if is_generic: tp_dict.__bases__ = (typing.Generic, dict) _maybe_adjust_parameters(tp_dict) + else: + # generic TypedDicts get __orig_bases__ from Generic + tp_dict.__orig_bases__ = bases or (TypedDict,) annotations = {} own_annotations = ns.get('__annotations__', {}) @@ -872,9 +1067,6 @@ else: if hasattr(typing, "Required"): get_type_hints = typing.get_type_hints else: - import functools - import types - # replaces _strip_annotations() def _strip_extras(t): """Strips Annotated, Required and NotRequired from a given type.""" @@ -887,12 +1079,12 @@ else: if stripped_args == t.__args__: return t return t.copy_with(stripped_args) - if hasattr(types, "GenericAlias") and isinstance(t, types.GenericAlias): + if hasattr(_types, "GenericAlias") and isinstance(t, _types.GenericAlias): stripped_args = tuple(_strip_extras(a) for a in t.__args__) if stripped_args == t.__args__: return t - return types.GenericAlias(t.__origin__, stripped_args) - if hasattr(types, "UnionType") and isinstance(t, types.UnionType): + return _types.GenericAlias(t.__origin__, stripped_args) + if hasattr(_types, "UnionType") and isinstance(t, _types.UnionType): stripped_args = tuple(_strip_extras(a) for a in t.__args__) if stripped_args == t.__args__: return t @@ -1155,42 +1347,62 @@ else: above.""") +def _set_default(type_param, default): + if isinstance(default, (tuple, list)): + type_param.__default__ = tuple((typing._type_check(d, "Default must be a type") + for d in default)) + elif default != _marker: + type_param.__default__ = typing._type_check(default, "Default must be a type") + else: + type_param.__default__ = None + + +def _set_module(typevarlike): + # for pickling: + def_mod = _caller(depth=3) + if def_mod != 'typing_extensions': + typevarlike.__module__ = def_mod + + class _DefaultMixin: """Mixin for TypeVarLike defaults.""" __slots__ = () + __init__ = _set_default - def __init__(self, default): - if isinstance(default, (tuple, list)): - self.__default__ = tuple((typing._type_check(d, "Default must be a type") - for d in default)) - elif default != _marker: - self.__default__ = typing._type_check(default, "Default must be a type") - else: - self.__default__ = None + +# Classes using this metaclass must provide a _backported_typevarlike ClassVar +class _TypeVarLikeMeta(type): + def __instancecheck__(cls, __instance: Any) -> bool: + return isinstance(__instance, cls._backported_typevarlike) # Add default and infer_variance parameters from PEP 696 and 695 -class TypeVar(typing.TypeVar, _DefaultMixin, _root=True): +class TypeVar(metaclass=_TypeVarLikeMeta): """Type variable.""" - __module__ = 'typing' + _backported_typevarlike = typing.TypeVar - def __init__(self, name, *constraints, bound=None, - covariant=False, contravariant=False, - default=_marker, infer_variance=False): - super().__init__(name, *constraints, bound=bound, covariant=covariant, - contravariant=contravariant) - _DefaultMixin.__init__(self, default) - self.__infer_variance__ = infer_variance + def __new__(cls, name, *constraints, bound=None, + covariant=False, contravariant=False, + default=_marker, infer_variance=False): + if hasattr(typing, "TypeAliasType"): + # PEP 695 implemented, can pass infer_variance to typing.TypeVar + typevar = typing.TypeVar(name, *constraints, bound=bound, + covariant=covariant, contravariant=contravariant, + infer_variance=infer_variance) + else: + typevar = typing.TypeVar(name, *constraints, bound=bound, + covariant=covariant, contravariant=contravariant) + if infer_variance and (covariant or contravariant): + raise ValueError("Variance cannot be specified with infer_variance.") + typevar.__infer_variance__ = infer_variance + _set_default(typevar, default) + _set_module(typevar) + return typevar - # for pickling: - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - def_mod = None - if def_mod != 'typing_extensions': - self.__module__ = def_mod + def __init_subclass__(cls) -> None: + raise TypeError(f"type '{__name__}.TypeVar' is not an acceptable base type") # Python 3.10+ has PEP 612 @@ -1258,25 +1470,33 @@ else: # 3.10+ if hasattr(typing, 'ParamSpec'): - # Add default Parameter - PEP 696 - class ParamSpec(typing.ParamSpec, _DefaultMixin, _root=True): - """Parameter specification variable.""" + # Add default parameter - PEP 696 + class ParamSpec(metaclass=_TypeVarLikeMeta): + """Parameter specification.""" - __module__ = 'typing' + _backported_typevarlike = typing.ParamSpec - def __init__(self, name, *, bound=None, covariant=False, contravariant=False, - default=_marker): - super().__init__(name, bound=bound, covariant=covariant, - contravariant=contravariant) - _DefaultMixin.__init__(self, default) + def __new__(cls, name, *, bound=None, + covariant=False, contravariant=False, + infer_variance=False, default=_marker): + if hasattr(typing, "TypeAliasType"): + # PEP 695 implemented, can pass infer_variance to typing.TypeVar + paramspec = typing.ParamSpec(name, bound=bound, + covariant=covariant, + contravariant=contravariant, + infer_variance=infer_variance) + else: + paramspec = typing.ParamSpec(name, bound=bound, + covariant=covariant, + contravariant=contravariant) + paramspec.__infer_variance__ = infer_variance - # for pickling: - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - def_mod = None - if def_mod != 'typing_extensions': - self.__module__ = def_mod + _set_default(paramspec, default) + _set_module(paramspec) + return paramspec + + def __init_subclass__(cls) -> None: + raise TypeError(f"type '{__name__}.ParamSpec' is not an acceptable base type") # 3.7-3.9 else: @@ -1341,11 +1561,12 @@ else: return ParamSpecKwargs(self) def __init__(self, name, *, bound=None, covariant=False, contravariant=False, - default=_marker): + infer_variance=False, default=_marker): super().__init__([self]) self.__name__ = name self.__covariant__ = bool(covariant) self.__contravariant__ = bool(contravariant) + self.__infer_variance__ = bool(infer_variance) if bound: self.__bound__ = typing._type_check(bound, 'Bound must be a type.') else: @@ -1353,15 +1574,14 @@ else: _DefaultMixin.__init__(self, default) # for pickling: - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - def_mod = None + def_mod = _caller() if def_mod != 'typing_extensions': self.__module__ = def_mod def __repr__(self): - if self.__covariant__: + if self.__infer_variance__: + prefix = '' + elif self.__covariant__: prefix = '+' elif self.__contravariant__: prefix = '-' @@ -1436,7 +1656,7 @@ def _concatenate_getitem(self, parameters): # 3.10+ if hasattr(typing, 'Concatenate'): Concatenate = typing.Concatenate - _ConcatenateGenericAlias = typing._ConcatenateGenericAlias # noqa + _ConcatenateGenericAlias = typing._ConcatenateGenericAlias # noqa: F811 # 3.9 elif sys.version_info[:2] >= (3, 9): @_TypeAliasForm @@ -1793,10 +2013,60 @@ else: """) -if hasattr(typing, "Unpack"): # 3.11+ +_UNPACK_DOC = """\ +Type unpack operator. + +The type unpack operator takes the child types from some container type, +such as `tuple[int, str]` or a `TypeVarTuple`, and 'pulls them out'. For +example: + + # For some generic class `Foo`: + Foo[Unpack[tuple[int, str]]] # Equivalent to Foo[int, str] + + Ts = TypeVarTuple('Ts') + # Specifies that `Bar` is generic in an arbitrary number of types. + # (Think of `Ts` as a tuple of an arbitrary number of individual + # `TypeVar`s, which the `Unpack` is 'pulling out' directly into the + # `Generic[]`.) + class Bar(Generic[Unpack[Ts]]): ... + Bar[int] # Valid + Bar[int, str] # Also valid + +From Python 3.11, this can also be done using the `*` operator: + + Foo[*tuple[int, str]] + class Bar(Generic[*Ts]): ... + +The operator can also be used along with a `TypedDict` to annotate +`**kwargs` in a function signature. For instance: + + class Movie(TypedDict): + name: str + year: int + + # This function expects two keyword arguments - *name* of type `str` and + # *year* of type `int`. + def foo(**kwargs: Unpack[Movie]): ... + +Note that there is only some runtime checking of this operator. Not +everything the runtime allows may be accepted by static type checkers. + +For more information, see PEP 646 and PEP 692. +""" + + +if sys.version_info >= (3, 12): # PEP 692 changed the repr of Unpack[] Unpack = typing.Unpack + + def _is_unpack(obj): + return get_origin(obj) is Unpack + elif sys.version_info[:2] >= (3, 9): class _UnpackSpecialForm(typing._SpecialForm, _root=True): + def __init__(self, getitem): + super().__init__(getitem) + self.__doc__ = _UNPACK_DOC + def __repr__(self): return 'typing_extensions.' + self._name @@ -1805,16 +2075,6 @@ elif sys.version_info[:2] >= (3, 9): @_UnpackSpecialForm def Unpack(self, parameters): - """A special typing construct to unpack a variadic type. For example: - - Shape = TypeVarTuple('Shape') - Batch = NewType('Batch', int) - - def add_batch_axis( - x: Array[Unpack[Shape]] - ) -> Array[Batch, Unpack[Shape]]: ... - - """ item = typing._type_check(parameters, f'{self._name} accepts only a single type.') return _UnpackAlias(self, (item,)) @@ -1834,18 +2094,7 @@ else: f'{self._name} accepts only a single type.') return _UnpackAlias(self, (item,)) - Unpack = _UnpackForm( - 'Unpack', - doc="""A special typing construct to unpack a variadic type. For example: - - Shape = TypeVarTuple('Shape') - Batch = NewType('Batch', int) - - def add_batch_axis( - x: Array[Unpack[Shape]] - ) -> Array[Batch, Unpack[Shape]]: ... - - """) + Unpack = _UnpackForm('Unpack', doc=_UNPACK_DOC) def _is_unpack(obj): return isinstance(obj, _UnpackAlias) @@ -1853,21 +2102,20 @@ else: if hasattr(typing, "TypeVarTuple"): # 3.11+ - # Add default Parameter - PEP 696 - class TypeVarTuple(typing.TypeVarTuple, _DefaultMixin, _root=True): + # Add default parameter - PEP 696 + class TypeVarTuple(metaclass=_TypeVarLikeMeta): """Type variable tuple.""" - def __init__(self, name, *, default=_marker): - super().__init__(name) - _DefaultMixin.__init__(self, default) + _backported_typevarlike = typing.TypeVarTuple - # for pickling: - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - def_mod = None - if def_mod != 'typing_extensions': - self.__module__ = def_mod + def __new__(cls, name, *, default=_marker): + tvt = typing.TypeVarTuple(name) + _set_default(tvt, default) + _set_module(tvt) + return tvt + + def __init_subclass__(self, *args, **kwds): + raise TypeError("Cannot subclass special typing classes") else: class TypeVarTuple(_DefaultMixin): @@ -1925,10 +2173,7 @@ else: _DefaultMixin.__init__(self, default) # for pickling: - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - def_mod = None + def_mod = _caller() if def_mod != 'typing_extensions': self.__module__ = def_mod @@ -2163,7 +2408,15 @@ else: When this decorator is applied to an object, the type checker will generate a diagnostic on usage of the deprecated object. - No runtime warning is issued. The decorator sets the ``__deprecated__`` + The warning specified by ``category`` will be emitted on use + of deprecated objects. For functions, that happens on calls; + for classes, on instantiation. If the ``category`` is ``None``, + no warning is emitted. The ``stacklevel`` determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + + The decorator sets the ``__deprecated__`` attribute on the decorated object to the deprecation message passed to the decorator. If applied to an overload, the decorator must be after the ``@overload`` decorator for the attribute to @@ -2223,18 +2476,13 @@ if not hasattr(typing, "TypeVarTuple"): typing._check_generic = _check_generic -# Backport typing.NamedTuple as it exists in Python 3.11. +# Backport typing.NamedTuple as it exists in Python 3.12. # In 3.11, the ability to define generic `NamedTuple`s was supported. # This was explicitly disallowed in 3.9-3.10, and only half-worked in <=3.8. -if sys.version_info >= (3, 11): +# On 3.12, we added __orig_bases__ to call-based NamedTuples +if sys.version_info >= (3, 12): NamedTuple = typing.NamedTuple else: - def _caller(): - try: - return sys._getframe(2).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): # For platforms without _getframe() - return None - def _make_nmtuple(name, types, module, defaults=()): fields = [n for n, t in types] annotations = {n: typing._type_check(t, f"field {n} annotation must be a type") @@ -2294,7 +2542,9 @@ else: elif kwargs: raise TypeError("Either list of fields or keywords" " can be provided to NamedTuple, not both") - return _make_nmtuple(__typename, __fields, module=_caller()) + nt = _make_nmtuple(__typename, __fields, module=_caller()) + nt.__orig_bases__ = (NamedTuple,) + return nt NamedTuple.__doc__ = typing.NamedTuple.__doc__ _NamedTuple = type.__new__(_NamedTupleMeta, 'NamedTuple', (), {}) @@ -2310,3 +2560,255 @@ else: return (_NamedTuple,) NamedTuple.__mro_entries__ = _namedtuple_mro_entries + + +if hasattr(collections.abc, "Buffer"): + Buffer = collections.abc.Buffer +else: + class Buffer(abc.ABC): + """Base class for classes that implement the buffer protocol. + + The buffer protocol allows Python objects to expose a low-level + memory buffer interface. Before Python 3.12, it is not possible + to implement the buffer protocol in pure Python code, or even + to check whether a class implements the buffer protocol. In + Python 3.12 and higher, the ``__buffer__`` method allows access + to the buffer protocol from Python code, and the + ``collections.abc.Buffer`` ABC allows checking whether a class + implements the buffer protocol. + + To indicate support for the buffer protocol in earlier versions, + inherit from this ABC, either in a stub file or at runtime, + or use ABC registration. This ABC provides no methods, because + there is no Python-accessible methods shared by pre-3.12 buffer + classes. It is useful primarily for static checks. + + """ + + # As a courtesy, register the most common stdlib buffer classes. + Buffer.register(memoryview) + Buffer.register(bytearray) + Buffer.register(bytes) + + +# Backport of types.get_original_bases, available on 3.12+ in CPython +if hasattr(_types, "get_original_bases"): + get_original_bases = _types.get_original_bases +else: + def get_original_bases(__cls): + """Return the class's "original" bases prior to modification by `__mro_entries__`. + + Examples:: + + from typing import TypeVar, Generic + from pip._vendor.typing_extensions import NamedTuple, TypedDict + + T = TypeVar("T") + class Foo(Generic[T]): ... + class Bar(Foo[int], float): ... + class Baz(list[str]): ... + Eggs = NamedTuple("Eggs", [("a", int), ("b", str)]) + Spam = TypedDict("Spam", {"a": int, "b": str}) + + assert get_original_bases(Bar) == (Foo[int], float) + assert get_original_bases(Baz) == (list[str],) + assert get_original_bases(Eggs) == (NamedTuple,) + assert get_original_bases(Spam) == (TypedDict,) + assert get_original_bases(int) == (object,) + """ + try: + return __cls.__orig_bases__ + except AttributeError: + try: + return __cls.__bases__ + except AttributeError: + raise TypeError( + f'Expected an instance of type, not {type(__cls).__name__!r}' + ) from None + + +# NewType is a class on Python 3.10+, making it pickleable +# The error message for subclassing instances of NewType was improved on 3.11+ +if sys.version_info >= (3, 11): + NewType = typing.NewType +else: + class NewType: + """NewType creates simple unique types with almost zero + runtime overhead. NewType(name, tp) is considered a subtype of tp + by static type checkers. At runtime, NewType(name, tp) returns + a dummy callable that simply returns its argument. Usage:: + UserId = NewType('UserId', int) + def name_by_id(user_id: UserId) -> str: + ... + UserId('user') # Fails type check + name_by_id(42) # Fails type check + name_by_id(UserId(42)) # OK + num = UserId(5) + 1 # type: int + """ + + def __call__(self, obj): + return obj + + def __init__(self, name, tp): + self.__qualname__ = name + if '.' in name: + name = name.rpartition('.')[-1] + self.__name__ = name + self.__supertype__ = tp + def_mod = _caller() + if def_mod != 'typing_extensions': + self.__module__ = def_mod + + def __mro_entries__(self, bases): + # We defined __mro_entries__ to get a better error message + # if a user attempts to subclass a NewType instance. bpo-46170 + supercls_name = self.__name__ + + class Dummy: + def __init_subclass__(cls): + subcls_name = cls.__name__ + raise TypeError( + f"Cannot subclass an instance of NewType. " + f"Perhaps you were looking for: " + f"`{subcls_name} = NewType({subcls_name!r}, {supercls_name})`" + ) + + return (Dummy,) + + def __repr__(self): + return f'{self.__module__}.{self.__qualname__}' + + def __reduce__(self): + return self.__qualname__ + + if sys.version_info >= (3, 10): + # PEP 604 methods + # It doesn't make sense to have these methods on Python <3.10 + + def __or__(self, other): + return typing.Union[self, other] + + def __ror__(self, other): + return typing.Union[other, self] + + +if hasattr(typing, "TypeAliasType"): + TypeAliasType = typing.TypeAliasType +else: + def _is_unionable(obj): + """Corresponds to is_unionable() in unionobject.c in CPython.""" + return obj is None or isinstance(obj, ( + type, + _types.GenericAlias, + _types.UnionType, + TypeAliasType, + )) + + class TypeAliasType: + """Create named, parameterized type aliases. + + This provides a backport of the new `type` statement in Python 3.12: + + type ListOrSet[T] = list[T] | set[T] + + is equivalent to: + + T = TypeVar("T") + ListOrSet = TypeAliasType("ListOrSet", list[T] | set[T], type_params=(T,)) + + The name ListOrSet can then be used as an alias for the type it refers to. + + The type_params argument should contain all the type parameters used + in the value of the type alias. If the alias is not generic, this + argument is omitted. + + Static type checkers should only support type aliases declared using + TypeAliasType that follow these rules: + + - The first argument (the name) must be a string literal. + - The TypeAliasType instance must be immediately assigned to a variable + of the same name. (For example, 'X = TypeAliasType("Y", int)' is invalid, + as is 'X, Y = TypeAliasType("X", int), TypeAliasType("Y", int)'). + + """ + + def __init__(self, name: str, value, *, type_params=()): + if not isinstance(name, str): + raise TypeError("TypeAliasType name must be a string") + self.__value__ = value + self.__type_params__ = type_params + + parameters = [] + for type_param in type_params: + if isinstance(type_param, TypeVarTuple): + parameters.extend(type_param) + else: + parameters.append(type_param) + self.__parameters__ = tuple(parameters) + def_mod = _caller() + if def_mod != 'typing_extensions': + self.__module__ = def_mod + # Setting this attribute closes the TypeAliasType from further modification + self.__name__ = name + + def __setattr__(self, __name: str, __value: object) -> None: + if hasattr(self, "__name__"): + self._raise_attribute_error(__name) + super().__setattr__(__name, __value) + + def __delattr__(self, __name: str) -> Never: + self._raise_attribute_error(__name) + + def _raise_attribute_error(self, name: str) -> Never: + # Match the Python 3.12 error messages exactly + if name == "__name__": + raise AttributeError("readonly attribute") + elif name in {"__value__", "__type_params__", "__parameters__", "__module__"}: + raise AttributeError( + f"attribute '{name}' of 'typing.TypeAliasType' objects " + "is not writable" + ) + else: + raise AttributeError( + f"'typing.TypeAliasType' object has no attribute '{name}'" + ) + + def __repr__(self) -> str: + return self.__name__ + + def __getitem__(self, parameters): + if not isinstance(parameters, tuple): + parameters = (parameters,) + parameters = [ + typing._type_check( + item, f'Subscripting {self.__name__} requires a type.' + ) + for item in parameters + ] + return typing._GenericAlias(self, tuple(parameters)) + + def __reduce__(self): + return self.__name__ + + def __init_subclass__(cls, *args, **kwargs): + raise TypeError( + "type 'typing_extensions.TypeAliasType' is not an acceptable base type" + ) + + # The presence of this method convinces typing._type_check + # that TypeAliasTypes are types. + def __call__(self): + raise TypeError("Type alias is not callable") + + if sys.version_info >= (3, 10): + def __or__(self, right): + # For forward compatibility with 3.12, reject Unions + # that are not accepted by the built-in Union. + if not _is_unionable(right): + return NotImplemented + return typing.Union[self, right] + + def __ror__(self, left): + if not _is_unionable(left): + return NotImplemented + return typing.Union[left, self] diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 732e90217..dcf89dc04 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -14,7 +14,7 @@ requests==2.31.0 urllib3==1.26.15 rich==13.3.3 pygments==2.14.0 - typing_extensions==4.5.0 + typing_extensions==4.6.0 resolvelib==1.0.1 setuptools==67.7.2 six==1.16.0 From 83f42685b79eff2e4b2154de8fcff694a4638da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 31 May 2023 09:13:33 +0200 Subject: [PATCH 021/216] Revert temporary fix for issue 9540 --- src/pip/__main__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/pip/__main__.py b/src/pip/__main__.py index fe34a7b77..599132611 100644 --- a/src/pip/__main__.py +++ b/src/pip/__main__.py @@ -1,6 +1,5 @@ import os import sys -import warnings # Remove '' and current working directory from the first entry # of sys.path, if present to avoid using current directory @@ -20,12 +19,6 @@ if __package__ == "": sys.path.insert(0, path) if __name__ == "__main__": - # Work around the error reported in #9540, pending a proper fix. - # Note: It is essential the warning filter is set *before* importing - # pip, as the deprecation happens at import time, not runtime. - warnings.filterwarnings( - "ignore", category=DeprecationWarning, module=".*packaging\\.version" - ) from pip._internal.cli.main import main as _main sys.exit(_main()) From 8d381eeec23cd1a0901266b189740d1a20f08534 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 2 Jun 2023 11:03:49 +0100 Subject: [PATCH 022/216] Warn if the --python option is not specified before the subcommand name --- news/12067.bugfix.rst | 1 + src/pip/_internal/cli/base_command.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 news/12067.bugfix.rst diff --git a/news/12067.bugfix.rst b/news/12067.bugfix.rst new file mode 100644 index 000000000..84f2d235e --- /dev/null +++ b/news/12067.bugfix.rst @@ -0,0 +1 @@ +Warn if the ``--python`` option is specified after the subcommand name. diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 637fba18c..87e6cf6de 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -131,6 +131,18 @@ class Command(CommandContextMixIn): ", ".join(sorted(always_enabled_features)), ) + + # Make sure that the --python argument isn't specified after the + # subcommand. We can tell, because if --python was specified, + # we should only reach this point if we're running in the created + # subprocess, which has the _PIP_RUNNING_IN_SUBPROCESS environment + # variable set. + if options.python and "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ: + logger.warning( + "The --python option is ignored if placed after " + "the pip subcommand name" + ) + # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. # This also affects isolated builds and it should. From 6aa4d48b23e11f6a837b61f4c3fda1a247758357 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 11:53:43 +0100 Subject: [PATCH 023/216] Fix the test failure --- tests/functional/test_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 637128274..8559d9368 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1157,7 +1157,7 @@ def test_install_nonlocal_compatible_wheel( "--find-links", data.find_links, "--only-binary=:all:", - "--python", + "--python-version", "3", "--platform", "fakeplat", @@ -1177,7 +1177,7 @@ def test_install_nonlocal_compatible_wheel( "--find-links", data.find_links, "--only-binary=:all:", - "--python", + "--python-version", "3", "--platform", "fakeplat", From 073666e2994896a66eff3a474dac3cf03b2dfdd9 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 11:57:04 +0100 Subject: [PATCH 024/216] Make this an error rather than a warning --- news/12067.bugfix.rst | 2 +- src/pip/_internal/cli/base_command.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/news/12067.bugfix.rst b/news/12067.bugfix.rst index 84f2d235e..87d76bc2b 100644 --- a/news/12067.bugfix.rst +++ b/news/12067.bugfix.rst @@ -1 +1 @@ -Warn if the ``--python`` option is specified after the subcommand name. +Fail with an error if the ``--python`` option is specified after the subcommand name. diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 87e6cf6de..5130a4505 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -138,10 +138,10 @@ class Command(CommandContextMixIn): # subprocess, which has the _PIP_RUNNING_IN_SUBPROCESS environment # variable set. if options.python and "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ: - logger.warning( - "The --python option is ignored if placed after " - "the pip subcommand name" + logger.critical( + "The --python option must be placed before the pip subcommand name" ) + sys.exit(ERROR) # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. From 16f145d30657093dab2fe66cd3695d248c46db45 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 12:03:22 +0100 Subject: [PATCH 025/216] Add a test for the error --- tests/functional/test_python_option.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/functional/test_python_option.py b/tests/functional/test_python_option.py index 8bf16d7a5..ca124933e 100644 --- a/tests/functional/test_python_option.py +++ b/tests/functional/test_python_option.py @@ -39,3 +39,14 @@ def test_python_interpreter( script.pip("--python", env_path, "uninstall", "simplewheel", "--yes") result = script.pip("--python", env_path, "list", "--format=json") assert json.loads(result.stdout) == before + +def test_error_python_option_wrong_location( + script: PipTestEnvironment, + tmpdir: Path, + shared_data: TestData, +) -> None: + env_path = os.fspath(tmpdir / "venv") + env = EnvBuilder(with_pip=False) + env.create(env_path) + + script.pip("list", "--python", env_path, "--format=json", expect_error=True) From de8f0b5ed17fc59fab1f20e9cca92c24ca89136d Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 12:11:41 +0100 Subject: [PATCH 026/216] Lint --- src/pip/_internal/cli/base_command.py | 1 - tests/functional/test_python_option.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 5130a4505..6a3b8e6c2 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -131,7 +131,6 @@ class Command(CommandContextMixIn): ", ".join(sorted(always_enabled_features)), ) - # Make sure that the --python argument isn't specified after the # subcommand. We can tell, because if --python was specified, # we should only reach this point if we're running in the created diff --git a/tests/functional/test_python_option.py b/tests/functional/test_python_option.py index ca124933e..ecfd819eb 100644 --- a/tests/functional/test_python_option.py +++ b/tests/functional/test_python_option.py @@ -40,6 +40,7 @@ def test_python_interpreter( result = script.pip("--python", env_path, "list", "--format=json") assert json.loads(result.stdout) == before + def test_error_python_option_wrong_location( script: PipTestEnvironment, tmpdir: Path, From 80cb6f443fca2c51b19ec4c3853b6261d2f4b6e6 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 14:26:05 +0100 Subject: [PATCH 027/216] Switch to ruff for linting --- .pre-commit-config.yaml | 13 ++++--------- noxfile.py | 2 +- pyproject.toml | 30 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fc455b9d..dd2fc6235 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,16 +21,11 @@ repos: hooks: - id: black -- repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.0.270 hooks: - - id: flake8 - additional_dependencies: [ - 'flake8-bugbear', - 'flake8-logging-format', - 'flake8-implicit-str-concat', - ] - exclude: tests/data + - id: ruff - repo: https://github.com/PyCQA/isort rev: 5.12.0 diff --git a/noxfile.py b/noxfile.py index 565a50399..ee03447d3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -219,7 +219,7 @@ def vendoring(session: nox.Session) -> None: new_version = old_version for inner_name, inner_version in pinned_requirements(vendor_txt): if inner_name == name: - # this is a dedicated assignment, to make flake8 happy + # this is a dedicated assignment, to make lint happy new_version = inner_version break else: diff --git a/pyproject.toml b/pyproject.toml index 139c37e18..18c990ef4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,3 +71,33 @@ setuptools = "pkg_resources" CacheControl = "https://raw.githubusercontent.com/ionrock/cachecontrol/v0.12.6/LICENSE.txt" distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt" webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE" + +[tool.ruff] +exclude = [ + "./build", + ".nox", + ".tox", + ".scratch", + "_vendor", + "data", +] +ignore = [ + "B019", + "B020", + "B904", # Ruff enables opinionated warnings by default + "B905", # Ruff enables opinionated warnings by default + "G202", +] +line-length = 88 +select = [ + "B", + "E", + "F", + "W", + "G", + "ISC", +] + +[tool.ruff.per-file-ignores] +"noxfile.py" = ["G"] +"tests/*" = ["B011"] From 9824a426d466680d132aa027add88ec0dda9116f Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 14:32:44 +0100 Subject: [PATCH 028/216] Fix new lint errors --- src/pip/_internal/network/auth.py | 4 +++- tests/lib/__init__.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index c0efa765c..94a82fa66 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -514,7 +514,9 @@ class MultiDomainBasicAuth(AuthBase): # Consume content and release the original connection to allow our new # request to reuse the same one. - resp.content + # The result of the assignment isn't used, it's just needed to consume + # the content. + _ = resp.content resp.raw.release_conn() # Add our new username and password to the request diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 7410072f5..2c6dbafb9 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -684,7 +684,9 @@ class PipTestEnvironment(TestFileEnvironment): # Pass expect_stderr=True to allow any stderr. We do this because # we do our checking of stderr further on in check_stderr(). kw["expect_stderr"] = True - result = super().run(cwd=cwd, *args, **kw) + # Ignore linter check + # B026 Star-arg unpacking after a keyword argument is strongly discouraged + result = super().run(cwd=cwd, *args, **kw) # noqa if expect_error and not allow_error: if result.returncode == 0: From 0af7d6de7adfdc2a0c3b5854cac8bccca3f7e887 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 14:36:34 +0100 Subject: [PATCH 029/216] Tidy up file exclusions --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 18c990ef4..5f5dfa641 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,10 +73,8 @@ distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt" webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE" [tool.ruff] -exclude = [ +extend-exclude = [ "./build", - ".nox", - ".tox", ".scratch", "_vendor", "data", From 7c3418b2d03aeae665a0a0f01b584ffba4bc57ea Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 14:50:10 +0100 Subject: [PATCH 030/216] Add explicit ID to noqa comment Co-authored-by: q0w <43147888+q0w@users.noreply.github.com> --- tests/lib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 2c6dbafb9..7c06feaf3 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -686,7 +686,7 @@ class PipTestEnvironment(TestFileEnvironment): kw["expect_stderr"] = True # Ignore linter check # B026 Star-arg unpacking after a keyword argument is strongly discouraged - result = super().run(cwd=cwd, *args, **kw) # noqa + result = super().run(cwd=cwd, *args, **kw) # noqa: B026 if expect_error and not allow_error: if result.returncode == 0: From bdef9159bd3734381fe90a907750c8e168170c38 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 5 Jun 2023 15:06:11 +0100 Subject: [PATCH 031/216] Use ruff for import sorting as well --- .pre-commit-config.yaml | 6 ------ pyproject.toml | 7 +++++++ tests/functional/test_install_compat.py | 8 ++++++-- tests/functional/test_install_upgrade.py | 9 +++++++-- tests/functional/test_install_user.py | 2 +- tests/functional/test_install_vcs_git.py | 2 +- tests/functional/test_wheel.py | 7 +++++-- 7 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd2fc6235..b0aef0d60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,12 +27,6 @@ repos: hooks: - id: ruff -- repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - files: \.py$ - - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.961 hooks: diff --git a/pyproject.toml b/pyproject.toml index 5f5dfa641..b7c0d1545 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,8 +94,15 @@ select = [ "W", "G", "ISC", + "I", ] [tool.ruff.per-file-ignores] "noxfile.py" = ["G"] "tests/*" = ["B011"] + +[tool.ruff.isort] +# We need to explicitly make pip "first party" as it's imported by code in +# the docs and tests directories. +known-first-party = ["pip"] +known-third-party = ["pip._vendor"] diff --git a/tests/functional/test_install_compat.py b/tests/functional/test_install_compat.py index ae27ebd53..8374d487b 100644 --- a/tests/functional/test_install_compat.py +++ b/tests/functional/test_install_compat.py @@ -7,8 +7,12 @@ from pathlib import Path import pytest -from tests.lib import pyversion # noqa: F401 -from tests.lib import PipTestEnvironment, TestData, assert_all_changes +from tests.lib import ( + PipTestEnvironment, + TestData, + assert_all_changes, + pyversion, # noqa: F401 +) @pytest.mark.network diff --git a/tests/functional/test_install_upgrade.py b/tests/functional/test_install_upgrade.py index fc61d70bc..09c01d7eb 100644 --- a/tests/functional/test_install_upgrade.py +++ b/tests/functional/test_install_upgrade.py @@ -6,8 +6,13 @@ from pathlib import Path import pytest -from tests.lib import pyversion # noqa: F401 -from tests.lib import PipTestEnvironment, ResolverVariant, TestData, assert_all_changes +from tests.lib import ( + PipTestEnvironment, + ResolverVariant, + TestData, + assert_all_changes, + pyversion, # noqa: F401 +) from tests.lib.local_repos import local_checkout from tests.lib.wheel import make_wheel diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 9bdadb942..3cae4a467 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -8,12 +8,12 @@ from pathlib import Path import pytest -from tests.lib import pyversion # noqa: F401 from tests.lib import ( PipTestEnvironment, TestData, create_basic_wheel_for_package, need_svn, + pyversion, # noqa: F401 ) from tests.lib.local_repos import local_checkout from tests.lib.venv import VirtualEnvironment diff --git a/tests/functional/test_install_vcs_git.py b/tests/functional/test_install_vcs_git.py index 971526c51..2abc7aa0f 100644 --- a/tests/functional/test_install_vcs_git.py +++ b/tests/functional/test_install_vcs_git.py @@ -3,11 +3,11 @@ from typing import Optional import pytest -from tests.lib import pyversion # noqa: F401 from tests.lib import ( PipTestEnvironment, _change_test_package_version, _create_test_package, + pyversion, # noqa: F401 ) from tests.lib.git_submodule_helpers import ( _change_test_package_submodule, diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index 1e3e90e41..cfaef541d 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -7,8 +7,11 @@ from pathlib import Path import pytest from pip._internal.cli.status_codes import ERROR -from tests.lib import pyversion # noqa: F401 -from tests.lib import PipTestEnvironment, TestData +from tests.lib import ( + PipTestEnvironment, + TestData, + pyversion, # noqa: F401 +) def add_files_to_dist_directory(folder: Path) -> None: From 16df8cdcbf895f707fe6f83bdceed8a09315018b Mon Sep 17 00:00:00 2001 From: Robert Pollak Date: Wed, 7 Jun 2023 16:04:20 +0200 Subject: [PATCH 032/216] NEWS.rst typo 'setttings' should be 'settings'. --- NEWS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index b0ae64263..f24aaaa40 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -53,7 +53,7 @@ Deprecations and Removals ``--config-settings``. (`#11859 `_) - Using ``--config-settings`` with projects that don't have a ``pyproject.toml`` now prints a deprecation warning. In the future the presence of config settings will automatically - enable the default build backend for legacy projects and pass the setttings to it. (`#11915 `_) + enable the default build backend for legacy projects and pass the settings to it. (`#11915 `_) - Remove ``setup.py install`` fallback when building a wheel failed for projects without ``pyproject.toml``. (`#8368 `_) - When the ``wheel`` package is not installed, pip now uses the default build backend From 6c3db098ff0b6e537157eff53b1aba79fa7fa4b0 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 20:58:40 +0100 Subject: [PATCH 033/216] Fix parsing of JSON index dist-info-metadata values --- src/pip/_internal/models/link.py | 97 +++++++++++++++++++++----------- tests/unit/test_collector.py | 37 +++++++----- 2 files changed, 89 insertions(+), 45 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index e741c3283..ee3045166 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -69,18 +69,6 @@ class LinkHash: def __post_init__(self) -> None: assert self.name in _SUPPORTED_HASHES - @classmethod - def parse_pep658_hash(cls, dist_info_metadata: str) -> Optional["LinkHash"]: - """Parse a PEP 658 data-dist-info-metadata hash.""" - if dist_info_metadata == "true": - return None - name, sep, value = dist_info_metadata.partition("=") - if not sep: - return None - if name not in _SUPPORTED_HASHES: - return None - return cls(name=name, value=value) - @classmethod @functools.lru_cache(maxsize=None) def find_hash_url_fragment(cls, url: str) -> Optional["LinkHash"]: @@ -107,6 +95,20 @@ class LinkHash: return hashes.is_hash_allowed(self.name, hex_digest=self.value) +@dataclass(frozen=True) +class MetadataFile: + """Information about a core metadata file associated with a distribution.""" + + hashes: Optional[dict[str, str]] + + # TODO: Do we care about stripping out unsupported hash methods? + def __init__(self, hashes: Optional[dict[str, str]]): + if hashes: + hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} + # We need to use this as this is a frozen dataclass + object.__setattr__(self, "hashes", hashes) + + def _clean_url_path_part(part: str) -> str: """ Clean a "part" of a URL path (i.e. after splitting on "@" characters). @@ -179,7 +181,7 @@ class Link(KeyBasedCompareMixin): "comes_from", "requires_python", "yanked_reason", - "dist_info_metadata", + "metadata_file_data", "cache_link_parsing", "egg_fragment", ] @@ -190,7 +192,7 @@ class Link(KeyBasedCompareMixin): comes_from: Optional[Union[str, "IndexContent"]] = None, requires_python: Optional[str] = None, yanked_reason: Optional[str] = None, - dist_info_metadata: Optional[str] = None, + metadata_file_data: Optional[MetadataFile] = None, cache_link_parsing: bool = True, hashes: Optional[Mapping[str, str]] = None, ) -> None: @@ -208,11 +210,10 @@ class Link(KeyBasedCompareMixin): a simple repository HTML link. If the file has been yanked but no reason was provided, this should be the empty string. See PEP 592 for more information and the specification. - :param dist_info_metadata: the metadata attached to the file, or None if no such - metadata is provided. This is the value of the "data-dist-info-metadata" - attribute, if present, in a simple repository HTML link. This may be parsed - into its own `Link` by `self.metadata_link()`. See PEP 658 for more - information and the specification. + :param metadata_file_data: the metadata attached to the file, or None if + no such metadata is provided. This argument, if not None, indicates + that a separate metadata file exists, and also optionally supplies + hashes for that file. :param cache_link_parsing: A flag that is used elsewhere to determine whether resources retrieved from this link should be cached. PyPI URLs should generally have this set to False, for example. @@ -220,6 +221,10 @@ class Link(KeyBasedCompareMixin): determine the validity of a download. """ + # The comes_from, requires_python, and metadata_file_data arguments are + # only used by classmethods of this class, and are not used in client + # code directly. + # url can be a UNC windows share if url.startswith("\\\\"): url = path_to_url(url) @@ -239,7 +244,7 @@ class Link(KeyBasedCompareMixin): self.comes_from = comes_from self.requires_python = requires_python if requires_python else None self.yanked_reason = yanked_reason - self.dist_info_metadata = dist_info_metadata + self.metadata_file_data = metadata_file_data super().__init__(key=url, defining_class=Link) @@ -262,9 +267,20 @@ class Link(KeyBasedCompareMixin): url = _ensure_quoted_url(urllib.parse.urljoin(page_url, file_url)) pyrequire = file_data.get("requires-python") yanked_reason = file_data.get("yanked") - dist_info_metadata = file_data.get("dist-info-metadata") hashes = file_data.get("hashes", {}) + # The dist-info-metadata value may be a boolean, or a dict of hashes. + metadata_info = file_data.get("dist-info-metadata", False) + if isinstance(metadata_info, dict): + # The file exists, and hashes have been supplied + metadata_file_data = MetadataFile(metadata_info) + elif metadata_info: + # The file exists, but there are no hashes + metadata_file_data = MetadataFile(None) + else: + # The file does not exist + metadata_file_data = None + # The Link.yanked_reason expects an empty string instead of a boolean. if yanked_reason and not isinstance(yanked_reason, str): yanked_reason = "" @@ -278,7 +294,7 @@ class Link(KeyBasedCompareMixin): requires_python=pyrequire, yanked_reason=yanked_reason, hashes=hashes, - dist_info_metadata=dist_info_metadata, + metadata_file_data=metadata_file_data, ) @classmethod @@ -298,14 +314,35 @@ class Link(KeyBasedCompareMixin): url = _ensure_quoted_url(urllib.parse.urljoin(base_url, href)) pyrequire = anchor_attribs.get("data-requires-python") yanked_reason = anchor_attribs.get("data-yanked") - dist_info_metadata = anchor_attribs.get("data-dist-info-metadata") + + # The dist-info-metadata value may be the string "true", or a string of + # the form "hashname=hashval" + metadata_info = anchor_attribs.get("data-dist-info-metadata") + if metadata_info == "true": + # The file exists, but there are no hashes + metadata_file_data = MetadataFile(None) + elif metadata_info is None: + # The file does not exist + metadata_file_data = None + else: + # The file exists, and hashes have been supplied + hashname, sep, hashval = metadata_info.partition("=") + if sep == "=": + metadata_file_data = MetadataFile({hashname: hashval}) + else: + # Error - data is wrong. Treat as no hashes supplied. + logger.debug( + "Index returned invalid data-dist-info-metadata value: %s", + metadata_info, + ) + metadata_file_data = MetadataFile(None) return cls( url, comes_from=page_url, requires_python=pyrequire, yanked_reason=yanked_reason, - dist_info_metadata=dist_info_metadata, + metadata_file_data=metadata_file_data, ) def __str__(self) -> str: @@ -407,17 +444,13 @@ class Link(KeyBasedCompareMixin): return match.group(1) def metadata_link(self) -> Optional["Link"]: - """Implementation of PEP 658 parsing.""" - # Note that Link.from_element() parsing the "data-dist-info-metadata" attribute - # from an HTML anchor tag is typically how the Link.dist_info_metadata attribute - # gets set. - if self.dist_info_metadata is None: + """Return a link to the associated core metadata file (if any).""" + if self.metadata_file_data is None: return None metadata_url = f"{self.url_without_fragment}.metadata" - metadata_link_hash = LinkHash.parse_pep658_hash(self.dist_info_metadata) - if metadata_link_hash is None: + if self.metadata_file_data.hashes is None: return Link(metadata_url) - return Link(metadata_url, hashes=metadata_link_hash.as_dict()) + return Link(metadata_url, hashes=self.metadata_file_data.hashes) def as_hashes(self) -> Hashes: return Hashes({k: [v] for k, v in self._hashes.items()}) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index e855d78e1..b3c9fcf1f 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -30,6 +30,7 @@ from pip._internal.models.index import PyPI from pip._internal.models.link import ( Link, LinkHash, + MetadataFile, _clean_url_path, _ensure_quoted_url, ) @@ -527,7 +528,7 @@ def test_parse_links_json() -> None: requires_python=">=3.7", yanked_reason=None, hashes={"sha256": "sha256 hash", "blake2b": "blake2b hash"}, - dist_info_metadata="sha512=aabdd41", + metadata_file_data=MetadataFile({"sha512": "aabdd41"}), ), ] @@ -603,12 +604,12 @@ _pkg1_requirement = Requirement("pkg1==1.0") ), ], ) -def test_parse_links__dist_info_metadata( +def test_parse_links__metadata_file_data( anchor_html: str, expected: Optional[str], hashes: Dict[str, str], ) -> None: - link = _test_parse_links_data_attribute(anchor_html, "dist_info_metadata", expected) + link = _test_parse_links_data_attribute(anchor_html, "metadata_file_data", expected) assert link._hashes == hashes @@ -1080,17 +1081,27 @@ def test_link_hash_parsing(url: str, result: Optional[LinkHash]) -> None: @pytest.mark.parametrize( - "dist_info_metadata, result", + "metadata_attrib, expected", [ - ("sha256=aa113592bbe", LinkHash("sha256", "aa113592bbe")), - ("sha256=", LinkHash("sha256", "")), - ("sha500=aa113592bbe", None), - ("true", None), - ("", None), - ("aa113592bbe", None), + ("sha256=aa113592bbe", MetadataFile({"sha256": "aa113592bbe"})), + ("sha256=", MetadataFile({"sha256": ""})), + ("sha500=aa113592bbe", MetadataFile({})), + ("true", MetadataFile(None)), + (None, None), + # TODO: Are these correct? + ("", MetadataFile(None)), + ("aa113592bbe", MetadataFile(None)), ], ) -def test_pep658_hash_parsing( - dist_info_metadata: str, result: Optional[LinkHash] +def test_metadata_file_info_parsing_html( + metadata_attrib: str, expected: Optional[MetadataFile] ) -> None: - assert LinkHash.parse_pep658_hash(dist_info_metadata) == result + attribs: Dict[str, Optional[str]] = { + "href": "something", + "data-dist-info-metadata": metadata_attrib, + } + page_url = "dummy_for_comes_from" + base_url = "https://index.url/simple" + link = Link.from_element(attribs, page_url, base_url) + assert link is not None and link.metadata_file_data == expected + # TODO: Do we need to do something for the JSON data? From cc554edab8897749c87495333018754080d06781 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 21:01:10 +0100 Subject: [PATCH 034/216] Add a news file --- news/12042.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12042.bugfix.rst diff --git a/news/12042.bugfix.rst b/news/12042.bugfix.rst new file mode 100644 index 000000000..34d977435 --- /dev/null +++ b/news/12042.bugfix.rst @@ -0,0 +1 @@ +Correctly parse ``dist-info-metadata`` values from JSON-format index data. From 8f89997d0dad1644b258297e2e3b9cc70d44e51d Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 21:11:34 +0100 Subject: [PATCH 035/216] Fix types to be 3.7-compatible --- src/pip/_internal/models/link.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index ee3045166..9630448bc 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -99,10 +99,10 @@ class LinkHash: class MetadataFile: """Information about a core metadata file associated with a distribution.""" - hashes: Optional[dict[str, str]] + hashes: Optional[Dict[str, str]] # TODO: Do we care about stripping out unsupported hash methods? - def __init__(self, hashes: Optional[dict[str, str]]): + def __init__(self, hashes: Optional[Dict[str, str]]): if hashes: hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} # We need to use this as this is a frozen dataclass From cfb4923d5d016dc58dc4e4b896992c476f0ddce8 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 21:21:32 +0100 Subject: [PATCH 036/216] Fix bad test data in test_parse_links_json --- tests/unit/test_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index b3c9fcf1f..838dd2efb 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -492,7 +492,7 @@ def test_parse_links_json() -> None: "url": "/files/holygrail-1.0-py3-none-any.whl", "hashes": {"sha256": "sha256 hash", "blake2b": "blake2b hash"}, "requires-python": ">=3.7", - "dist-info-metadata": "sha512=aabdd41", + "dist-info-metadata": {"sha512": "aabdd41"}, }, ], } From 93b274eee79b9c114728f0864a29751ee7698fca Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 7 Jun 2023 21:44:48 +0100 Subject: [PATCH 037/216] Missed a change to one of the tests --- tests/unit/test_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index 838dd2efb..513e4b134 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -587,19 +587,19 @@ _pkg1_requirement = Requirement("pkg1==1.0") # Test with value "true". ( '', - "true", + MetadataFile(None), {}, ), # Test with a provided hash value. ( '', # noqa: E501 - "sha256=aa113592bbe", + MetadataFile({"sha256": "aa113592bbe"}), {}, ), # Test with a provided hash value for both the requirement as well as metadata. ( '', # noqa: E501 - "sha256=aa113592bbe", + MetadataFile({"sha256": "aa113592bbe"}), {"sha512": "abc132409cb"}, ), ], From 232cc9dd5284fbc7554bcd291bf14e31413da78a Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 8 Jun 2023 09:52:51 +0100 Subject: [PATCH 038/216] Parse hash data before passing to MetadataFile --- src/pip/_internal/models/link.py | 24 ++++++++++++++++-------- tests/unit/test_collector.py | 5 ++--- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 9630448bc..7b45f3f3e 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -101,12 +101,20 @@ class MetadataFile: hashes: Optional[Dict[str, str]] - # TODO: Do we care about stripping out unsupported hash methods? - def __init__(self, hashes: Optional[Dict[str, str]]): - if hashes: - hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} - # We need to use this as this is a frozen dataclass - object.__setattr__(self, "hashes", hashes) + def __post_init__(self) -> None: + if self.hashes is not None: + assert all(name in _SUPPORTED_HASHES for name in self.hashes) + + +def supported_hashes(hashes: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]: + # Remove any unsupported hash types from the mapping. If this leaves no + # supported hashes, return None + if hashes is None: + return None + hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} + if len(hashes) > 0: + return hashes + return None def _clean_url_path_part(part: str) -> str: @@ -273,7 +281,7 @@ class Link(KeyBasedCompareMixin): metadata_info = file_data.get("dist-info-metadata", False) if isinstance(metadata_info, dict): # The file exists, and hashes have been supplied - metadata_file_data = MetadataFile(metadata_info) + metadata_file_data = MetadataFile(supported_hashes(metadata_info)) elif metadata_info: # The file exists, but there are no hashes metadata_file_data = MetadataFile(None) @@ -328,7 +336,7 @@ class Link(KeyBasedCompareMixin): # The file exists, and hashes have been supplied hashname, sep, hashval = metadata_info.partition("=") if sep == "=": - metadata_file_data = MetadataFile({hashname: hashval}) + metadata_file_data = MetadataFile(supported_hashes({hashname: hashval})) else: # Error - data is wrong. Treat as no hashes supplied. logger.debug( diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index 513e4b134..d1e68fab7 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -1085,10 +1085,10 @@ def test_link_hash_parsing(url: str, result: Optional[LinkHash]) -> None: [ ("sha256=aa113592bbe", MetadataFile({"sha256": "aa113592bbe"})), ("sha256=", MetadataFile({"sha256": ""})), - ("sha500=aa113592bbe", MetadataFile({})), + ("sha500=aa113592bbe", MetadataFile(None)), ("true", MetadataFile(None)), (None, None), - # TODO: Are these correct? + # Attribute is present but invalid ("", MetadataFile(None)), ("aa113592bbe", MetadataFile(None)), ], @@ -1104,4 +1104,3 @@ def test_metadata_file_info_parsing_html( base_url = "https://index.url/simple" link = Link.from_element(attribs, page_url, base_url) assert link is not None and link.metadata_file_data == expected - # TODO: Do we need to do something for the JSON data? From 5168881b438b2851ae4c9459a8c06beee2058639 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 8 Jun 2023 10:10:15 +0100 Subject: [PATCH 039/216] Implement PEP 714 - rename dist-info-metadata --- src/pip/_internal/models/link.py | 19 +++++++++--- tests/unit/test_collector.py | 53 +++++++++++++++++++++++++++++--- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 7b45f3f3e..3cfc3e8c4 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -277,8 +277,13 @@ class Link(KeyBasedCompareMixin): yanked_reason = file_data.get("yanked") hashes = file_data.get("hashes", {}) - # The dist-info-metadata value may be a boolean, or a dict of hashes. - metadata_info = file_data.get("dist-info-metadata", False) + # PEP 714: Indexes must use the name core-metadata, but + # clients should support the old name as a fallback for compatibility. + metadata_info = file_data.get("core-metadata") + if metadata_info is None: + metadata_info = file_data.get("dist-info-metadata") + + # The metadata info value may be a boolean, or a dict of hashes. if isinstance(metadata_info, dict): # The file exists, and hashes have been supplied metadata_file_data = MetadataFile(supported_hashes(metadata_info)) @@ -286,7 +291,7 @@ class Link(KeyBasedCompareMixin): # The file exists, but there are no hashes metadata_file_data = MetadataFile(None) else: - # The file does not exist + # False or not present: the file does not exist metadata_file_data = None # The Link.yanked_reason expects an empty string instead of a boolean. @@ -323,9 +328,13 @@ class Link(KeyBasedCompareMixin): pyrequire = anchor_attribs.get("data-requires-python") yanked_reason = anchor_attribs.get("data-yanked") - # The dist-info-metadata value may be the string "true", or a string of + # PEP 714: Indexes must use the name data-core-metadata, but + # clients should support the old name as a fallback for compatibility. + metadata_info = anchor_attribs.get("data-core-metadata") + if metadata_info is None: + metadata_info = anchor_attribs.get("data-dist-info-metadata") + # The metadata info value may be the string "true", or a string of # the form "hashname=hashval" - metadata_info = anchor_attribs.get("data-dist-info-metadata") if metadata_info == "true": # The file exists, but there are no hashes metadata_file_data = MetadataFile(None) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index d1e68fab7..5410a4afc 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -486,7 +486,15 @@ def test_parse_links_json() -> None: "requires-python": ">=3.7", "dist-info-metadata": False, }, - # Same as above, but parsing dist-info-metadata. + # Same as above, but parsing core-metadata. + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "sha256 hash", "blake2b": "blake2b hash"}, + "requires-python": ">=3.7", + "core-metadata": {"sha512": "aabdd41"}, + }, + # Ensure fallback to dist-info-metadata works { "filename": "holygrail-1.0-py3-none-any.whl", "url": "/files/holygrail-1.0-py3-none-any.whl", @@ -494,6 +502,15 @@ def test_parse_links_json() -> None: "requires-python": ">=3.7", "dist-info-metadata": {"sha512": "aabdd41"}, }, + # Ensure that core-metadata gets priority. + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "sha256 hash", "blake2b": "blake2b hash"}, + "requires-python": ">=3.7", + "core-metadata": {"sha512": "aabdd41"}, + "dist-info-metadata": {"sha512": "this_is_wrong"}, + }, ], } ).encode("utf8") @@ -530,6 +547,22 @@ def test_parse_links_json() -> None: hashes={"sha256": "sha256 hash", "blake2b": "blake2b hash"}, metadata_file_data=MetadataFile({"sha512": "aabdd41"}), ), + Link( + "https://example.com/files/holygrail-1.0-py3-none-any.whl", + comes_from=page.url, + requires_python=">=3.7", + yanked_reason=None, + hashes={"sha256": "sha256 hash", "blake2b": "blake2b hash"}, + metadata_file_data=MetadataFile({"sha512": "aabdd41"}), + ), + Link( + "https://example.com/files/holygrail-1.0-py3-none-any.whl", + comes_from=page.url, + requires_python=">=3.7", + yanked_reason=None, + hashes={"sha256": "sha256 hash", "blake2b": "blake2b hash"}, + metadata_file_data=MetadataFile({"sha512": "aabdd41"}), + ), ] # Ensure the metadata info can be parsed into the correct link. @@ -586,22 +619,34 @@ _pkg1_requirement = Requirement("pkg1==1.0") ), # Test with value "true". ( - '', + '', MetadataFile(None), {}, ), # Test with a provided hash value. ( - '', # noqa: E501 + '', # noqa: E501 MetadataFile({"sha256": "aa113592bbe"}), {}, ), # Test with a provided hash value for both the requirement as well as metadata. ( - '', # noqa: E501 + '', # noqa: E501 MetadataFile({"sha256": "aa113592bbe"}), {"sha512": "abc132409cb"}, ), + # Ensure the fallback to the old name works. + ( + '', # noqa: E501 + MetadataFile({"sha256": "aa113592bbe"}), + {}, + ), + # Ensure that the data-core-metadata name gets priority. + ( + '', # noqa: E501 + MetadataFile({"sha256": "aa113592bbe"}), + {}, + ), ], ) def test_parse_links__metadata_file_data( From 3eb3ddd873dfde2c146fcc5c82b2fa0ba363ac69 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Thu, 8 Jun 2023 17:57:17 +0200 Subject: [PATCH 040/216] Fix slowness on Python 3.11 when updating an existing large environment. --- news/12079.bugfix.rst | 1 + src/pip/_internal/resolution/resolvelib/candidates.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 news/12079.bugfix.rst diff --git a/news/12079.bugfix.rst b/news/12079.bugfix.rst new file mode 100644 index 000000000..79496798a --- /dev/null +++ b/news/12079.bugfix.rst @@ -0,0 +1 @@ +Fix slowness on Python 3.11 when updating an existing large environment. diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 31020e27a..de04e1d73 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -341,6 +341,7 @@ class AlreadyInstalledCandidate(Candidate): self.dist = dist self._ireq = _make_install_req_from_dist(dist, template) self._factory = factory + self._version = None # This is just logging some messages, so we can do it eagerly. # The returned dist would be exactly the same as self.dist because we @@ -376,7 +377,9 @@ class AlreadyInstalledCandidate(Candidate): @property def version(self) -> CandidateVersion: - return self.dist.version + if self._version is None: + self._version = self.dist.version + return self._version @property def is_editable(self) -> bool: From 1269d0d240f4d22ed1134bb854bf2177a82a8d66 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Mon, 12 Jun 2023 16:31:28 +0200 Subject: [PATCH 041/216] Update news snippet. --- news/12079.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/12079.bugfix.rst b/news/12079.bugfix.rst index 79496798a..d65c8ee76 100644 --- a/news/12079.bugfix.rst +++ b/news/12079.bugfix.rst @@ -1 +1 @@ -Fix slowness on Python 3.11 when updating an existing large environment. +Fix slowness when using ``importlib.metadata`` and there is a large overlap between already installed and to-be-installed packages. From 1a80e41504008c8f3b63b5fb59c6b7476fcae3b9 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Mon, 12 Jun 2023 21:43:34 +0200 Subject: [PATCH 042/216] Mention in changelog that 12079 is an issue mostly in Python 3.11. --- news/12079.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/12079.bugfix.rst b/news/12079.bugfix.rst index d65c8ee76..b5b7e553b 100644 --- a/news/12079.bugfix.rst +++ b/news/12079.bugfix.rst @@ -1 +1 @@ -Fix slowness when using ``importlib.metadata`` and there is a large overlap between already installed and to-be-installed packages. +Fix slowness when using ``importlib.metadata`` and there is a large overlap between already installed and to-be-installed packages. This is the default in Python 3.11, though it can be overridden with the ``_PIP_USE_IMPORTLIB_METADATA`` environment variable. From 6aef9326d93dee61cb69cfa8770ca445891fa990 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Mon, 12 Jun 2023 21:49:12 +0200 Subject: [PATCH 043/216] Update news/12079.bugfix.rst Co-authored-by: Tzu-ping Chung --- news/12079.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/12079.bugfix.rst b/news/12079.bugfix.rst index b5b7e553b..5ee050268 100644 --- a/news/12079.bugfix.rst +++ b/news/12079.bugfix.rst @@ -1 +1 @@ -Fix slowness when using ``importlib.metadata`` and there is a large overlap between already installed and to-be-installed packages. This is the default in Python 3.11, though it can be overridden with the ``_PIP_USE_IMPORTLIB_METADATA`` environment variable. +Fix slowness when using ``importlib.metadata`` (the default way for pip to read metadata in Python 3.11+) and there is a large overlap between already installed and to-be-installed packages. From 67deaf7576a9aa14f37bd6ebef5bdce038069b60 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Wed, 14 Jun 2023 22:36:15 +0800 Subject: [PATCH 044/216] Add permission check before configuration --- src/pip/_internal/configuration.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 8fd46c9b8..35790369a 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -210,8 +210,16 @@ class Configuration: # Ensure directory exists. ensure_dir(os.path.dirname(fname)) - with open(fname, "w") as f: - parser.write(f) + # Ensure directory's permission(need to be writeable) + if os.access(fname, os.W_OK): + with open(fname, "w") as f: + parser.write(f) + else: + raise ConfigurationError( + "Configuation file not writeable".format( + ": ".join(fname) + ) + ) # # Private routines From 19b41050efdcb2b8e74c16fded3064abb96f95fa Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Wed, 14 Jun 2023 23:00:03 +0800 Subject: [PATCH 045/216] Add permission check before configuration --- news/11920.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11920.bugfix.rst diff --git a/news/11920.bugfix.rst b/news/11920.bugfix.rst new file mode 100644 index 000000000..f91667c52 --- /dev/null +++ b/news/11920.bugfix.rst @@ -0,0 +1 @@ +Add permission check before configuration \ No newline at end of file From dafca3f7e3985ce2f2351d168efbd91fa5b2f4b2 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:11:20 +0800 Subject: [PATCH 046/216] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 35790369a..f8de92397 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,10 +216,8 @@ class Configuration: parser.write(f) else: raise ConfigurationError( - "Configuation file not writeable".format( - ": ".join(fname) + "Configuation file not writeable {}".format(': '.join(fname)) ) - ) # # Private routines From f77661e478d3169e43fd2ba540a0c9778e924a45 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:35:46 +0800 Subject: [PATCH 047/216] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index f8de92397..ebcfe7df0 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -211,10 +211,10 @@ class Configuration: ensure_dir(os.path.dirname(fname)) # Ensure directory's permission(need to be writeable) - if os.access(fname, os.W_OK): + try: with open(fname, "w") as f: parser.write(f) - else: + except: raise ConfigurationError( "Configuation file not writeable {}".format(': '.join(fname)) ) From f74650725b7fa57a5b17f2b839624fa36622ba7e Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Wed, 14 Jun 2023 22:36:15 +0800 Subject: [PATCH 048/216] Add permission check before configuration --- src/pip/_internal/configuration.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 8fd46c9b8..35790369a 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -210,8 +210,16 @@ class Configuration: # Ensure directory exists. ensure_dir(os.path.dirname(fname)) - with open(fname, "w") as f: - parser.write(f) + # Ensure directory's permission(need to be writeable) + if os.access(fname, os.W_OK): + with open(fname, "w") as f: + parser.write(f) + else: + raise ConfigurationError( + "Configuation file not writeable".format( + ": ".join(fname) + ) + ) # # Private routines From 920bcd0c631be649317b5a1e00019ad029353d5c Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Wed, 14 Jun 2023 23:00:03 +0800 Subject: [PATCH 049/216] Add permission check before configuration --- news/11920.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11920.bugfix.rst diff --git a/news/11920.bugfix.rst b/news/11920.bugfix.rst new file mode 100644 index 000000000..f91667c52 --- /dev/null +++ b/news/11920.bugfix.rst @@ -0,0 +1 @@ +Add permission check before configuration \ No newline at end of file From 2dbda58efc6bfd0b9115626d294ab95b11712e9a Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:11:20 +0800 Subject: [PATCH 050/216] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 35790369a..f8de92397 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,10 +216,8 @@ class Configuration: parser.write(f) else: raise ConfigurationError( - "Configuation file not writeable".format( - ": ".join(fname) + "Configuation file not writeable {}".format(': '.join(fname)) ) - ) # # Private routines From 17147b8fd36a851da7897da26eef8f68aec364a0 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:35:46 +0800 Subject: [PATCH 051/216] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index f8de92397..ebcfe7df0 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -211,10 +211,10 @@ class Configuration: ensure_dir(os.path.dirname(fname)) # Ensure directory's permission(need to be writeable) - if os.access(fname, os.W_OK): + try: with open(fname, "w") as f: parser.write(f) - else: + except: raise ConfigurationError( "Configuation file not writeable {}".format(': '.join(fname)) ) From 5986dd27c5797245c70daf922f89bddc16ee656e Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Fri, 16 Jun 2023 21:35:49 +0800 Subject: [PATCH 052/216] Add permission check before configuration --- src/pip/_internal/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index ebcfe7df0..f8de92397 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -211,10 +211,10 @@ class Configuration: ensure_dir(os.path.dirname(fname)) # Ensure directory's permission(need to be writeable) - try: + if os.access(fname, os.W_OK): with open(fname, "w") as f: parser.write(f) - except: + else: raise ConfigurationError( "Configuation file not writeable {}".format(': '.join(fname)) ) From 05e936aecb6427403c6a99dcbcefa63514b98425 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Sat, 17 Jun 2023 12:16:25 +0800 Subject: [PATCH 053/216] Add permission check before configuration7 --- src/pip/_internal/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index f8de92397..ba34886b6 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -345,7 +345,7 @@ class Configuration: # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) + ) or not os.access(config_file[kinds.SITE], os.W_OK) if should_load_user_config: # The legacy config file is overridden by the new config file yield kinds.USER, config_files[kinds.USER] From 8747268d44250b164957f3a7671bd30ef9f250c7 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Sat, 17 Jun 2023 12:30:36 +0800 Subject: [PATCH 054/216] Add permission check before configuration8 --- src/pip/_internal/configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index ba34886b6..0a7d183ae 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -345,7 +345,8 @@ class Configuration: # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) or not os.access(config_file[kinds.SITE], os.W_OK) + ) or not os.access(config_files[kinds.SITE], os.W_OK) + if should_load_user_config: # The legacy config file is overridden by the new config file yield kinds.USER, config_files[kinds.USER] From 81c8a3ffbca1aca2c7bfae1433d0dd52050fc9b3 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Sat, 17 Jun 2023 12:42:47 +0800 Subject: [PATCH 055/216] Add permission check before configuration9 --- src/pip/_internal/configuration.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 0a7d183ae..5cd889d39 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -342,10 +342,24 @@ class Configuration: # at the base we have any global configuration yield kinds.GLOBAL, config_files[kinds.GLOBAL] + site_accessable = int + site_index = 0 + site_all_accessable = bool + + for fname in config_files[kinds.SITE]: + site_index += 1 + if os.access(fname, os.W_OK): + site_accessable += 1 + + if site_accessable < site_index: + site_all_accessable = False + elif site_accessable == site_index: + site_all_accessable = True + # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) or not os.access(config_files[kinds.SITE], os.W_OK) + ) or not site_all_accessable == True if should_load_user_config: # The legacy config file is overridden by the new config file From 54be97e05ce6e303dddb95f9d8d0291c35f7189c Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Mon, 15 Aug 2022 16:43:58 -0700 Subject: [PATCH 056/216] Use strict optional checking in glibc Suggested by pradyunsg in #11374 `--no-strict-optional` defeats half the purpose of using mypy. This change is trivial, we already catch AttributeError in the case that mypy is concerned about. --- src/pip/_internal/utils/glibc.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index 7bd3c2068..2f64f5aa3 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -1,6 +1,3 @@ -# The following comment should be removed at some point in the future. -# mypy: strict-optional=False - import os import sys from typing import Optional, Tuple @@ -21,7 +18,7 @@ def glibc_version_string_confstr() -> Optional[str]: return None try: # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": - _, version = os.confstr("CS_GNU_LIBC_VERSION").split() + _, version = os.confstr("CS_GNU_LIBC_VERSION").split() # type: ignore[union-attr] except (AttributeError, OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None From b5a40ed64bf534858f9cd43d0a9fcd86e478c836 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 01:45:24 -0700 Subject: [PATCH 057/216] news --- news/5C12428B-09FA-49BC-A886-6F5D8885BC14.trivial.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 news/5C12428B-09FA-49BC-A886-6F5D8885BC14.trivial.rst diff --git a/news/5C12428B-09FA-49BC-A886-6F5D8885BC14.trivial.rst b/news/5C12428B-09FA-49BC-A886-6F5D8885BC14.trivial.rst new file mode 100644 index 000000000..e69de29bb From 42117756313b1b59c3a6b4f637795688fc36b19e Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 01:48:49 -0700 Subject: [PATCH 058/216] remove the error code to silence ruff --- src/pip/_internal/utils/glibc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index 2f64f5aa3..d0e1dbc2c 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -18,7 +18,7 @@ def glibc_version_string_confstr() -> Optional[str]: return None try: # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": - _, version = os.confstr("CS_GNU_LIBC_VERSION").split() # type: ignore[union-attr] + _, version = os.confstr("CS_GNU_LIBC_VERSION").split() # type: ignore except (AttributeError, OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None From 36014e6f495bd57363e935e466da8f165acd51f6 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 01:51:56 -0700 Subject: [PATCH 059/216] don't catch attributeerror --- src/pip/_internal/utils/glibc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index d0e1dbc2c..b9dbfbcbf 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -17,9 +17,12 @@ def glibc_version_string_confstr() -> Optional[str]: if sys.platform == "win32": return None try: + gnu_libc_version = os.confstr("CS_GNU_LIBC_VERSION") + if gnu_libc_version is None: + return None # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": - _, version = os.confstr("CS_GNU_LIBC_VERSION").split() # type: ignore - except (AttributeError, OSError, ValueError): + _, version = gnu_libc_version + except (OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None return version From e995f2564495d0bb0cb609c0b48091c3f5708ed8 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 01:54:20 -0700 Subject: [PATCH 060/216] nope --- src/pip/_internal/utils/glibc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index b9dbfbcbf..b80c1881b 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -21,7 +21,7 @@ def glibc_version_string_confstr() -> Optional[str]: if gnu_libc_version is None: return None # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": - _, version = gnu_libc_version + _, version = gnu_libc_version.split() except (OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None From b5377aeb73ac1fabc45abb07ed92b209b3213e98 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 20 Jun 2023 02:04:36 -0700 Subject: [PATCH 061/216] nope --- src/pip/_internal/utils/glibc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py index b80c1881b..81342afa4 100644 --- a/src/pip/_internal/utils/glibc.py +++ b/src/pip/_internal/utils/glibc.py @@ -22,7 +22,7 @@ def glibc_version_string_confstr() -> Optional[str]: return None # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": _, version = gnu_libc_version.split() - except (OSError, ValueError): + except (AttributeError, OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return None return version From c57bad63da242068f74e1eb96c97c3bc800b7f88 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 22 Jun 2023 11:27:48 +0800 Subject: [PATCH 062/216] Add permission check before configuration10 --- src/pip/_internal/configuration.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 5cd889d39..a44f13321 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -211,12 +211,12 @@ class Configuration: ensure_dir(os.path.dirname(fname)) # Ensure directory's permission(need to be writeable) - if os.access(fname, os.W_OK): + try: with open(fname, "w") as f: parser.write(f) - else: + except IOError as error: raise ConfigurationError( - "Configuation file not writeable {}".format(': '.join(fname)) + "An error occurred while writing to the configuration file: {0}\nError message: {1}".format(fname, error) ) # @@ -342,24 +342,10 @@ class Configuration: # at the base we have any global configuration yield kinds.GLOBAL, config_files[kinds.GLOBAL] - site_accessable = int - site_index = 0 - site_all_accessable = bool - - for fname in config_files[kinds.SITE]: - site_index += 1 - if os.access(fname, os.W_OK): - site_accessable += 1 - - if site_accessable < site_index: - site_all_accessable = False - elif site_accessable == site_index: - site_all_accessable = True - # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) or not site_all_accessable == True + ) if should_load_user_config: # The legacy config file is overridden by the new config file From c4709d2b2251528647e19be0d0f8ea83d1011e24 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 22 Jun 2023 11:31:43 +0800 Subject: [PATCH 063/216] Add permission check before configuration11 --- src/pip/_internal/configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index a44f13321..869842e37 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,7 +216,8 @@ class Configuration: parser.write(f) except IOError as error: raise ConfigurationError( - "An error occurred while writing to the configuration file: {0}\nError message: {1}".format(fname, error) + "An error occurred while writing to the configuration file: {0}\n \ + Error message: {1}".format(fname, error) ) # From 7572dbc09581c139199496adc57fefb9f404a6b2 Mon Sep 17 00:00:00 2001 From: JasonMo1 <111677135+JasonMo1@users.noreply.github.com> Date: Thu, 22 Jun 2023 12:03:58 +0800 Subject: [PATCH 064/216] Add IO check before save configuration1 --- news/11920.bugfix.rst | 2 +- src/pip/_internal/configuration.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/news/11920.bugfix.rst b/news/11920.bugfix.rst index f91667c52..d8e22ee9b 100644 --- a/news/11920.bugfix.rst +++ b/news/11920.bugfix.rst @@ -1 +1 @@ -Add permission check before configuration \ No newline at end of file +Add permission check before configuration diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 869842e37..46562652f 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,8 +216,10 @@ class Configuration: parser.write(f) except IOError as error: raise ConfigurationError( - "An error occurred while writing to the configuration file: {0}\n \ - Error message: {1}".format(fname, error) + "An error occurred while writing to the configuration file: {0}\n \ + Error message: {1}".format( + fname, error + ) ) # @@ -346,7 +348,7 @@ class Configuration: # per-user configuration next should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) - ) + ) if should_load_user_config: # The legacy config file is overridden by the new config file From 9fa64244522a237725e95e36eaedcef4ac25e87c Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 25 Jun 2023 12:06:02 +0100 Subject: [PATCH 065/216] Remove the no-response workflow (#12102) --- .github/workflows/no-response.yml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .github/workflows/no-response.yml diff --git a/.github/workflows/no-response.yml b/.github/workflows/no-response.yml deleted file mode 100644 index 939290b93..000000000 --- a/.github/workflows/no-response.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: No Response - -# Both `issue_comment` and `scheduled` event types are required for this Action -# to work properly. -on: - issue_comment: - types: [created] - schedule: - # Schedule for five minutes after the hour, every hour - - cron: '5 * * * *' - -jobs: - noResponse: - runs-on: ubuntu-latest - steps: - - uses: lee-dohm/no-response@v0.5.0 - with: - token: ${{ github.token }} - responseRequiredLabel: "S: awaiting response" From 108c055f727e91b366b3f12854d1dccc39195d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 27 Jun 2023 15:10:38 +0200 Subject: [PATCH 066/216] Revert "xfail test_pip_wheel_ext_module_with_tmpdir_inside" This reverts commit fab519dfd936def55ed019a0da18d0d77509fb95. --- tests/functional/test_wheel.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index cfaef541d..c0e279492 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -343,15 +343,6 @@ def test_pip_wheel_with_user_set_in_config( sys.platform.startswith("win"), reason="The empty extension module does not work on Win", ) -@pytest.mark.xfail( - condition=sys.platform == "darwin" and sys.version_info < (3, 9), - reason=( - "Unexplained 'no module named platform' in " - "https://github.com/pypa/wheel/blob" - "/c87e6ed82b58b41b258a3e8c852af8bc1817bb00" - "/src/wheel/vendored/packaging/tags.py#L396-L411" - ), -) def test_pip_wheel_ext_module_with_tmpdir_inside( script: PipTestEnvironment, data: TestData, common_wheels: Path ) -> None: From c1ead0aa37d5f3526820fcbec1c89011c5063236 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 May 2022 11:14:16 -0400 Subject: [PATCH 067/216] Switch to new cache format and new cache location. --- news/2984.bugfix | 1 + src/pip/_internal/cli/req_command.py | 2 +- src/pip/_internal/commands/cache.py | 21 ++++++++++++------ src/pip/_internal/network/cache.py | 32 +++++++++++++++++++++------- tests/functional/test_cache.py | 4 ++-- 5 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 news/2984.bugfix diff --git a/news/2984.bugfix b/news/2984.bugfix new file mode 100644 index 000000000..d75974349 --- /dev/null +++ b/news/2984.bugfix @@ -0,0 +1 @@ +pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). \ No newline at end of file diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index c2f4e38be..c9c201959 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -123,7 +123,7 @@ class SessionCommandMixin(CommandContextMixIn): ssl_context = None session = PipSession( - cache=os.path.join(cache_dir, "http") if cache_dir else None, + cache=os.path.join(cache_dir, "http-v2") if cache_dir else None, retries=retries if retries is not None else options.retries, trusted_hosts=options.trusted_hosts, index_urls=self._get_index_urls(options), diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index e96d2b492..a11e151f3 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -93,17 +93,21 @@ class CacheCommand(Command): num_http_files = len(self._find_http_files(options)) num_packages = len(self._find_wheels(options, "*")) - http_cache_location = self._cache_dir(options, "http") + http_cache_location = self._cache_dir(options, "http-v2") + old_http_cache_location = self._cache_dir(options, "http") wheels_cache_location = self._cache_dir(options, "wheels") http_cache_size = filesystem.format_directory_size(http_cache_location) + old_http_cache_size = filesystem.format_directory_size(old_http_cache_location) wheels_cache_size = filesystem.format_directory_size(wheels_cache_location) message = ( textwrap.dedent( """ - Package index page cache location: {http_cache_location} - Package index page cache size: {http_cache_size} - Number of HTTP files: {num_http_files} + Package index page cache location (new): {http_cache_location} + Package index page cache location (old): {old_http_cache_location} + Package index page cache size (new): {http_cache_size} + Package index page cache size (old): {old_http_cache_size} + Number of HTTP files (old+new cache): {num_http_files} Locally built wheels location: {wheels_cache_location} Locally built wheels size: {wheels_cache_size} Number of locally built wheels: {package_count} @@ -111,7 +115,9 @@ class CacheCommand(Command): ) .format( http_cache_location=http_cache_location, + old_http_cache_location=old_http_cache_location, http_cache_size=http_cache_size, + old_http_cache_size=old_http_cache_size, num_http_files=num_http_files, wheels_cache_location=wheels_cache_location, package_count=num_packages, @@ -195,8 +201,11 @@ class CacheCommand(Command): return os.path.join(options.cache_dir, subdir) def _find_http_files(self, options: Values) -> List[str]: - http_dir = self._cache_dir(options, "http") - return filesystem.find_files(http_dir, "*") + old_http_dir = self._cache_dir(options, "http") + new_http_dir = self._cache_dir(options, "http-v2") + return filesystem.find_files(old_http_dir, "*") + filesystem.find_files( + new_http_dir, "*" + ) def _find_wheels(self, options: Values, pattern: str) -> List[str]: wheel_dir = self._cache_dir(options, "wheels") diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index a81a23985..b85be2e48 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -3,10 +3,10 @@ import os from contextlib import contextmanager -from typing import Generator, Optional +from typing import BinaryIO, Generator, Optional -from pip._vendor.cachecontrol.cache import BaseCache -from pip._vendor.cachecontrol.caches import FileCache +from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache +from pip._vendor.cachecontrol.caches import SeparateBodyFileCache from pip._vendor.requests.models import Response from pip._internal.utils.filesystem import adjacent_tmp_file, replace @@ -28,7 +28,7 @@ def suppressed_cache_errors() -> Generator[None, None, None]: pass -class SafeFileCache(BaseCache): +class SafeFileCache(SeparateBodyBaseCache): """ A file based cache which is safe to use even when the target directory may not be accessible or writable. @@ -43,7 +43,7 @@ class SafeFileCache(BaseCache): # From cachecontrol.caches.file_cache.FileCache._fn, brought into our # class for backwards-compatibility and to avoid using a non-public # method. - hashed = FileCache.encode(name) + hashed = SeparateBodyFileCache.encode(name) parts = list(hashed[:5]) + [hashed] return os.path.join(self.directory, *parts) @@ -53,17 +53,33 @@ class SafeFileCache(BaseCache): with open(path, "rb") as f: return f.read() - def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None: - path = self._get_cache_path(key) + def _write(self, path: str, data: bytes) -> None: with suppressed_cache_errors(): ensure_dir(os.path.dirname(path)) with adjacent_tmp_file(path) as f: - f.write(value) + f.write(data) replace(f.name, path) + def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None: + path = self._get_cache_path(key) + self._write(path, value) + def delete(self, key: str) -> None: path = self._get_cache_path(key) with suppressed_cache_errors(): os.remove(path) + os.remove(path + ".body") + + def get_body(self, key: str) -> Optional[BinaryIO]: + path = self._get_cache_path(key) + ".body" + with suppressed_cache_errors(): + return open(path, "rb") + + def set_body(self, key: str, body: Optional[bytes]) -> None: + if body is None: + # Workaround for https://github.com/ionrock/cachecontrol/issues/276 + return + path = self._get_cache_path(key) + ".body" + self._write(path, body) diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index 788abdd2b..5eea6a96e 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -20,7 +20,7 @@ def cache_dir(script: PipTestEnvironment) -> str: @pytest.fixture def http_cache_dir(cache_dir: str) -> str: - return os.path.normcase(os.path.join(cache_dir, "http")) + return os.path.normcase(os.path.join(cache_dir, "http-v2")) @pytest.fixture @@ -211,7 +211,7 @@ def test_cache_info( ) -> None: result = script.pip("cache", "info") - assert f"Package index page cache location: {http_cache_dir}" in result.stdout + assert f"Package index page cache location (new): {http_cache_dir}" in result.stdout assert f"Locally built wheels location: {wheel_cache_dir}" in result.stdout num_wheels = len(wheel_cache_files) assert f"Number of locally built wheels: {num_wheels}" in result.stdout From fa87c9eb23dd25ad5cb03fe480a3fc4b92deb7a6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 May 2022 13:06:38 -0400 Subject: [PATCH 068/216] Testing for body methods of network cache. --- src/pip/_internal/network/cache.py | 1 + tests/unit/test_network_cache.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index b85be2e48..11c76bf0f 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -70,6 +70,7 @@ class SafeFileCache(SeparateBodyBaseCache): path = self._get_cache_path(key) with suppressed_cache_errors(): os.remove(path) + with suppressed_cache_errors(): os.remove(path + ".body") def get_body(self, key: str) -> Optional[BinaryIO]: diff --git a/tests/unit/test_network_cache.py b/tests/unit/test_network_cache.py index a5519864f..88597e4c1 100644 --- a/tests/unit/test_network_cache.py +++ b/tests/unit/test_network_cache.py @@ -31,6 +31,14 @@ class TestSafeFileCache: cache.delete("test key") assert cache.get("test key") is None + def test_cache_roundtrip_body(self, cache_tmpdir: Path) -> None: + cache = SafeFileCache(os.fspath(cache_tmpdir)) + assert cache.get_body("test key") is None + cache.set_body("test key", b"a test string") + assert cache.get_body("test key").read() == b"a test string" + cache.delete("test key") + assert cache.get_body("test key") is None + @pytest.mark.skipif("sys.platform == 'win32'") def test_safe_get_no_perms( self, cache_tmpdir: Path, monkeypatch: pytest.MonkeyPatch From fde34fdf8416a9692c07a899d2668f3f6ccf9df7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 12:06:15 -0400 Subject: [PATCH 069/216] Temporary workaround for https://github.com/ionrock/cachecontrol/issues/276 until it's fixed upstream. --- src/pip/_vendor/cachecontrol/controller.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 7f23529f1..14ba62976 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -407,7 +407,17 @@ class CacheController(object): """ cache_url = self.cache_url(request.url) - cached_response = self.serializer.loads(request, self.cache.get(cache_url)) + # NOTE: This is a hot-patch for + # https://github.com/ionrock/cachecontrol/issues/276 until it's fixed + # upstream. + if isinstance(self.cache, SeparateBodyBaseCache): + body_file = self.cache.get_body(cache_url) + else: + body_file = None + + cached_response = self.serializer.loads( + request, self.cache.get(cache_url), body_file + ) if not cached_response: # we didn't have a cached response From 5b7c999581e1b892a8048f6bd1275e8501614911 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 12:10:05 -0400 Subject: [PATCH 070/216] Whitespace fix. --- news/2984.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/2984.bugfix b/news/2984.bugfix index d75974349..cce561815 100644 --- a/news/2984.bugfix +++ b/news/2984.bugfix @@ -1 +1 @@ -pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). \ No newline at end of file +pip uses less memory when caching large packages. As a result, there is a new on-disk cache format stored in a new directory ($PIP_CACHE_DIR/http-v2). From 7a609bfdd5a23d404124a0ace5e3598966fe2466 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 12:11:34 -0400 Subject: [PATCH 071/216] Mypy fix. --- tests/unit/test_network_cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_network_cache.py b/tests/unit/test_network_cache.py index 88597e4c1..d62d1ab69 100644 --- a/tests/unit/test_network_cache.py +++ b/tests/unit/test_network_cache.py @@ -35,7 +35,9 @@ class TestSafeFileCache: cache = SafeFileCache(os.fspath(cache_tmpdir)) assert cache.get_body("test key") is None cache.set_body("test key", b"a test string") - assert cache.get_body("test key").read() == b"a test string" + body = cache.get_body("test key") + assert body is not None + assert body.read() == b"a test string" cache.delete("test key") assert cache.get_body("test key") is None From 3dbba12132b55a937c095c1c5537baf8652533ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 12:12:29 -0400 Subject: [PATCH 072/216] Correct name. --- news/{2984.bugfix => 2984.bugfix.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{2984.bugfix => 2984.bugfix.rst} (100%) diff --git a/news/2984.bugfix b/news/2984.bugfix.rst similarity index 100% rename from news/2984.bugfix rename to news/2984.bugfix.rst From bff05e5622b1dcf66c1556fb421441086b93456c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 14:50:29 -0400 Subject: [PATCH 073/216] Switch to proposed upstream fix. --- src/pip/_internal/network/cache.py | 3 -- src/pip/_vendor/cachecontrol/controller.py | 57 +++++++++++----------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index 11c76bf0f..f52e9974f 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -79,8 +79,5 @@ class SafeFileCache(SeparateBodyBaseCache): return open(path, "rb") def set_body(self, key: str, body: Optional[bytes]) -> None: - if body is None: - # Workaround for https://github.com/ionrock/cachecontrol/issues/276 - return path = self._get_cache_path(key) + ".body" self._write(path, body) diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 14ba62976..7af0e002d 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -122,6 +122,26 @@ class CacheController(object): return retval + def _load_from_cache(self, request): + """ + Load a cached response, or return None if it's not available. + """ + cache_url = request.url + cache_data = self.cache.get(cache_url) + if cache_data is None: + logger.debug("No cache entry available") + return None + + if isinstance(self.cache, SeparateBodyBaseCache): + body_file = self.cache.get_body(cache_url) + else: + body_file = None + + result = self.serializer.loads(request, cache_data, body_file) + if result is None: + logger.warning("Cache entry deserialization failed, entry ignored") + return result + def cached_request(self, request): """ Return a cached response if it exists in the cache, otherwise @@ -140,21 +160,9 @@ class CacheController(object): logger.debug('Request header has "max_age" as 0, cache bypassed') return False - # Request allows serving from the cache, let's see if we find something - cache_data = self.cache.get(cache_url) - if cache_data is None: - logger.debug("No cache entry available") - return False - - if isinstance(self.cache, SeparateBodyBaseCache): - body_file = self.cache.get_body(cache_url) - else: - body_file = None - - # Check whether it can be deserialized - resp = self.serializer.loads(request, cache_data, body_file) + # Check whether we can load the response from the cache: + resp = self._load_from_cache(request) if not resp: - logger.warning("Cache entry deserialization failed, entry ignored") return False # If we have a cached permanent redirect, return it immediately. We @@ -240,8 +248,7 @@ class CacheController(object): return False def conditional_headers(self, request): - cache_url = self.cache_url(request.url) - resp = self.serializer.loads(request, self.cache.get(cache_url)) + resp = self._load_from_cache(request) new_headers = {} if resp: @@ -267,7 +274,10 @@ class CacheController(object): self.serializer.dumps(request, response, b""), expires=expires_time, ) - self.cache.set_body(cache_url, body) + # body is None can happen when, for example, we're only updating + # headers, as is the case in update_cached_response(). + if body is not None: + self.cache.set_body(cache_url, body) else: self.cache.set( cache_url, @@ -406,18 +416,7 @@ class CacheController(object): gotten a 304 as the response. """ cache_url = self.cache_url(request.url) - - # NOTE: This is a hot-patch for - # https://github.com/ionrock/cachecontrol/issues/276 until it's fixed - # upstream. - if isinstance(self.cache, SeparateBodyBaseCache): - body_file = self.cache.get_body(cache_url) - else: - body_file = None - - cached_response = self.serializer.loads( - request, self.cache.get(cache_url), body_file - ) + cached_response = self._load_from_cache(request) if not cached_response: # we didn't have a cached response From 46f9154daecffa52966f6e917f0819cfabb112ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 15:11:16 -0400 Subject: [PATCH 074/216] Make sure the file gets closed. --- tests/unit/test_network_cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_network_cache.py b/tests/unit/test_network_cache.py index d62d1ab69..aa849f3b0 100644 --- a/tests/unit/test_network_cache.py +++ b/tests/unit/test_network_cache.py @@ -37,7 +37,8 @@ class TestSafeFileCache: cache.set_body("test key", b"a test string") body = cache.get_body("test key") assert body is not None - assert body.read() == b"a test string" + with body: + assert body.read() == b"a test string" cache.delete("test key") assert cache.get_body("test key") is None From bada6316dfcb16d50f214b88f8d2424f0e9d990b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 May 2022 15:13:46 -0400 Subject: [PATCH 075/216] More accurate type. --- src/pip/_internal/network/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index f52e9974f..d6b8ccdcf 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -78,6 +78,6 @@ class SafeFileCache(SeparateBodyBaseCache): with suppressed_cache_errors(): return open(path, "rb") - def set_body(self, key: str, body: Optional[bytes]) -> None: + def set_body(self, key: str, body: bytes) -> None: path = self._get_cache_path(key) + ".body" self._write(path, body) From ca08c16b9e81ce21021831fb1bdfe3a76387fd25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Jun 2023 13:56:56 -0400 Subject: [PATCH 076/216] Vendor latest version of CacheControl. --- src/pip/_vendor/cachecontrol.pyi | 1 - src/pip/_vendor/cachecontrol/__init__.py | 18 ++- src/pip/_vendor/cachecontrol/_cmd.py | 24 ++-- src/pip/_vendor/cachecontrol/adapter.py | 83 +++++++---- src/pip/_vendor/cachecontrol/cache.py | 31 ++-- .../_vendor/cachecontrol/caches/__init__.py | 5 +- .../_vendor/cachecontrol/caches/file_cache.py | 78 +++++----- .../cachecontrol/caches/redis_cache.py | 29 ++-- src/pip/_vendor/cachecontrol/compat.py | 32 ----- src/pip/_vendor/cachecontrol/controller.py | 116 ++++++++++----- src/pip/_vendor/cachecontrol/filewrapper.py | 27 ++-- src/pip/_vendor/cachecontrol/heuristics.py | 54 ++++--- src/pip/_vendor/cachecontrol/py.typed | 0 src/pip/_vendor/cachecontrol/serialize.py | 135 ++++++++++++------ src/pip/_vendor/cachecontrol/wrapper.py | 33 +++-- src/pip/_vendor/vendor.txt | 2 +- 16 files changed, 405 insertions(+), 263 deletions(-) delete mode 100644 src/pip/_vendor/cachecontrol.pyi delete mode 100644 src/pip/_vendor/cachecontrol/compat.py create mode 100644 src/pip/_vendor/cachecontrol/py.typed diff --git a/src/pip/_vendor/cachecontrol.pyi b/src/pip/_vendor/cachecontrol.pyi deleted file mode 100644 index 636a66bac..000000000 --- a/src/pip/_vendor/cachecontrol.pyi +++ /dev/null @@ -1 +0,0 @@ -from cachecontrol import * \ No newline at end of file diff --git a/src/pip/_vendor/cachecontrol/__init__.py b/src/pip/_vendor/cachecontrol/__init__.py index f631ae6df..3701cdd6b 100644 --- a/src/pip/_vendor/cachecontrol/__init__.py +++ b/src/pip/_vendor/cachecontrol/__init__.py @@ -8,11 +8,21 @@ Make it easy to import from cachecontrol without long namespaces. """ __author__ = "Eric Larson" __email__ = "eric@ionrock.org" -__version__ = "0.12.11" +__version__ = "0.13.0" -from .wrapper import CacheControl -from .adapter import CacheControlAdapter -from .controller import CacheController +from pip._vendor.cachecontrol.adapter import CacheControlAdapter +from pip._vendor.cachecontrol.controller import CacheController +from pip._vendor.cachecontrol.wrapper import CacheControl + +__all__ = [ + "__author__", + "__email__", + "__version__", + "CacheControlAdapter", + "CacheController", + "CacheControl", +] import logging + logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/src/pip/_vendor/cachecontrol/_cmd.py b/src/pip/_vendor/cachecontrol/_cmd.py index 4266b5ee9..ab4dac3dd 100644 --- a/src/pip/_vendor/cachecontrol/_cmd.py +++ b/src/pip/_vendor/cachecontrol/_cmd.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 import logging +from argparse import ArgumentParser +from typing import TYPE_CHECKING from pip._vendor import requests @@ -10,16 +12,19 @@ from pip._vendor.cachecontrol.adapter import CacheControlAdapter from pip._vendor.cachecontrol.cache import DictCache from pip._vendor.cachecontrol.controller import logger -from argparse import ArgumentParser +if TYPE_CHECKING: + from argparse import Namespace + + from pip._vendor.cachecontrol.controller import CacheController -def setup_logging(): +def setup_logging() -> None: logger.setLevel(logging.DEBUG) handler = logging.StreamHandler() logger.addHandler(handler) -def get_session(): +def get_session() -> requests.Session: adapter = CacheControlAdapter( DictCache(), cache_etags=True, serializer=None, heuristic=None ) @@ -27,17 +32,17 @@ def get_session(): sess.mount("http://", adapter) sess.mount("https://", adapter) - sess.cache_controller = adapter.controller + sess.cache_controller = adapter.controller # type: ignore[attr-defined] return sess -def get_args(): +def get_args() -> "Namespace": parser = ArgumentParser() parser.add_argument("url", help="The URL to try and cache") return parser.parse_args() -def main(args=None): +def main() -> None: args = get_args() sess = get_session() @@ -48,10 +53,13 @@ def main(args=None): setup_logging() # try setting the cache - sess.cache_controller.cache_response(resp.request, resp.raw) + cache_controller: "CacheController" = ( + sess.cache_controller # type: ignore[attr-defined] + ) + cache_controller.cache_response(resp.request, resp.raw) # Now try to get it - if sess.cache_controller.cached_request(resp.request): + if cache_controller.cached_request(resp.request): print("Cached!") else: print("Not cached :(") diff --git a/src/pip/_vendor/cachecontrol/adapter.py b/src/pip/_vendor/cachecontrol/adapter.py index 94c75e1a0..83c08e003 100644 --- a/src/pip/_vendor/cachecontrol/adapter.py +++ b/src/pip/_vendor/cachecontrol/adapter.py @@ -2,15 +2,24 @@ # # SPDX-License-Identifier: Apache-2.0 -import types import functools +import types import zlib +from typing import TYPE_CHECKING, Any, Collection, Mapping, Optional, Tuple, Type, Union from pip._vendor.requests.adapters import HTTPAdapter -from .controller import CacheController, PERMANENT_REDIRECT_STATUSES -from .cache import DictCache -from .filewrapper import CallbackFileWrapper +from pip._vendor.cachecontrol.cache import DictCache +from pip._vendor.cachecontrol.controller import PERMANENT_REDIRECT_STATUSES, CacheController +from pip._vendor.cachecontrol.filewrapper import CallbackFileWrapper + +if TYPE_CHECKING: + from pip._vendor.requests import PreparedRequest, Response + from pip._vendor.urllib3 import HTTPResponse + + from pip._vendor.cachecontrol.cache import BaseCache + from pip._vendor.cachecontrol.heuristics import BaseHeuristic + from pip._vendor.cachecontrol.serialize import Serializer class CacheControlAdapter(HTTPAdapter): @@ -18,15 +27,15 @@ class CacheControlAdapter(HTTPAdapter): def __init__( self, - cache=None, - cache_etags=True, - controller_class=None, - serializer=None, - heuristic=None, - cacheable_methods=None, - *args, - **kw - ): + cache: Optional["BaseCache"] = None, + cache_etags: bool = True, + controller_class: Optional[Type[CacheController]] = None, + serializer: Optional["Serializer"] = None, + heuristic: Optional["BaseHeuristic"] = None, + cacheable_methods: Optional[Collection[str]] = None, + *args: Any, + **kw: Any, + ) -> None: super(CacheControlAdapter, self).__init__(*args, **kw) self.cache = DictCache() if cache is None else cache self.heuristic = heuristic @@ -37,7 +46,18 @@ class CacheControlAdapter(HTTPAdapter): self.cache, cache_etags=cache_etags, serializer=serializer ) - def send(self, request, cacheable_methods=None, **kw): + def send( + self, + request: "PreparedRequest", + stream: bool = False, + timeout: Union[None, float, Tuple[float, float], Tuple[float, None]] = None, + verify: Union[bool, str] = True, + cert: Union[ + None, bytes, str, Tuple[Union[bytes, str], Union[bytes, str]] + ] = None, + proxies: Optional[Mapping[str, str]] = None, + cacheable_methods: Optional[Collection[str]] = None, + ) -> "Response": """ Send a request. Use the request information to see if it exists in the cache and cache the response if we need to and can. @@ -54,13 +74,19 @@ class CacheControlAdapter(HTTPAdapter): # check for etags and add headers if appropriate request.headers.update(self.controller.conditional_headers(request)) - resp = super(CacheControlAdapter, self).send(request, **kw) + resp = super(CacheControlAdapter, self).send( + request, stream, timeout, verify, cert, proxies + ) return resp def build_response( - self, request, response, from_cache=False, cacheable_methods=None - ): + self, + request: "PreparedRequest", + response: "HTTPResponse", + from_cache: bool = False, + cacheable_methods: Optional[Collection[str]] = None, + ) -> "Response": """ Build a response by making a request or using the cache. @@ -102,36 +128,39 @@ class CacheControlAdapter(HTTPAdapter): else: # Wrap the response file with a wrapper that will cache the # response when the stream has been consumed. - response._fp = CallbackFileWrapper( - response._fp, + response._fp = CallbackFileWrapper( # type: ignore[attr-defined] + response._fp, # type: ignore[attr-defined] functools.partial( self.controller.cache_response, request, response ), ) if response.chunked: - super_update_chunk_length = response._update_chunk_length + super_update_chunk_length = response._update_chunk_length # type: ignore[attr-defined] - def _update_chunk_length(self): + def _update_chunk_length(self: "HTTPResponse") -> None: super_update_chunk_length() if self.chunk_left == 0: - self._fp._close() + self._fp._close() # type: ignore[attr-defined] - response._update_chunk_length = types.MethodType( + response._update_chunk_length = types.MethodType( # type: ignore[attr-defined] _update_chunk_length, response ) - resp = super(CacheControlAdapter, self).build_response(request, response) + resp: "Response" = super( # type: ignore[no-untyped-call] + CacheControlAdapter, self + ).build_response(request, response) # See if we should invalidate the cache. if request.method in self.invalidating_methods and resp.ok: + assert request.url is not None cache_url = self.controller.cache_url(request.url) self.cache.delete(cache_url) # Give the request a from_cache attr to let people use it - resp.from_cache = from_cache + resp.from_cache = from_cache # type: ignore[attr-defined] return resp - def close(self): + def close(self) -> None: self.cache.close() - super(CacheControlAdapter, self).close() + super(CacheControlAdapter, self).close() # type: ignore[no-untyped-call] diff --git a/src/pip/_vendor/cachecontrol/cache.py b/src/pip/_vendor/cachecontrol/cache.py index 2a965f595..61031d234 100644 --- a/src/pip/_vendor/cachecontrol/cache.py +++ b/src/pip/_vendor/cachecontrol/cache.py @@ -7,37 +7,43 @@ The cache object API for implementing caches. The default is a thread safe in-memory dictionary. """ from threading import Lock +from typing import IO, TYPE_CHECKING, MutableMapping, Optional, Union + +if TYPE_CHECKING: + from datetime import datetime class BaseCache(object): - - def get(self, key): + def get(self, key: str) -> Optional[bytes]: raise NotImplementedError() - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + ) -> None: raise NotImplementedError() - def delete(self, key): + def delete(self, key: str) -> None: raise NotImplementedError() - def close(self): + def close(self) -> None: pass class DictCache(BaseCache): - - def __init__(self, init_dict=None): + def __init__(self, init_dict: Optional[MutableMapping[str, bytes]] = None) -> None: self.lock = Lock() self.data = init_dict or {} - def get(self, key): + def get(self, key: str) -> Optional[bytes]: return self.data.get(key, None) - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + ) -> None: with self.lock: self.data.update({key: value}) - def delete(self, key): + def delete(self, key: str) -> None: with self.lock: if key in self.data: self.data.pop(key) @@ -55,10 +61,11 @@ class SeparateBodyBaseCache(BaseCache): Similarly, the body should be loaded separately via ``get_body()``. """ - def set_body(self, key, body): + + def set_body(self, key: str, body: bytes) -> None: raise NotImplementedError() - def get_body(self, key): + def get_body(self, key: str) -> Optional["IO[bytes]"]: """ Return the body as file-like object. """ diff --git a/src/pip/_vendor/cachecontrol/caches/__init__.py b/src/pip/_vendor/cachecontrol/caches/__init__.py index 37827291f..24ff469ff 100644 --- a/src/pip/_vendor/cachecontrol/caches/__init__.py +++ b/src/pip/_vendor/cachecontrol/caches/__init__.py @@ -2,8 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from .file_cache import FileCache, SeparateBodyFileCache -from .redis_cache import RedisCache - +from pip._vendor.cachecontrol.caches.file_cache import FileCache, SeparateBodyFileCache +from pip._vendor.cachecontrol.caches.redis_cache import RedisCache __all__ = ["FileCache", "SeparateBodyFileCache", "RedisCache"] diff --git a/src/pip/_vendor/cachecontrol/caches/file_cache.py b/src/pip/_vendor/cachecontrol/caches/file_cache.py index f1ddb2ebd..0437c4e8a 100644 --- a/src/pip/_vendor/cachecontrol/caches/file_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/file_cache.py @@ -5,18 +5,18 @@ import hashlib import os from textwrap import dedent +from typing import IO, TYPE_CHECKING, Optional, Type, Union -from ..cache import BaseCache, SeparateBodyBaseCache -from ..controller import CacheController +from pip._vendor.cachecontrol.cache import BaseCache, SeparateBodyBaseCache +from pip._vendor.cachecontrol.controller import CacheController -try: - FileNotFoundError -except NameError: - # py2.X - FileNotFoundError = (IOError, OSError) +if TYPE_CHECKING: + from datetime import datetime + + from filelock import BaseFileLock -def _secure_open_write(filename, fmode): +def _secure_open_write(filename: str, fmode: int) -> "IO[bytes]": # We only want to write to this file, so open it in write only mode flags = os.O_WRONLY @@ -62,37 +62,27 @@ class _FileCacheMixin: def __init__( self, - directory, - forever=False, - filemode=0o0600, - dirmode=0o0700, - use_dir_lock=None, - lock_class=None, - ): - - if use_dir_lock is not None and lock_class is not None: - raise ValueError("Cannot use use_dir_lock and lock_class together") - + directory: str, + forever: bool = False, + filemode: int = 0o0600, + dirmode: int = 0o0700, + lock_class: Optional[Type["BaseFileLock"]] = None, + ) -> None: try: - from lockfile import LockFile - from lockfile.mkdirlockfile import MkdirLockFile + if lock_class is None: + from filelock import FileLock + + lock_class = FileLock except ImportError: notice = dedent( """ NOTE: In order to use the FileCache you must have - lockfile installed. You can install it via pip: - pip install lockfile + filelock installed. You can install it via pip: + pip install filelock """ ) raise ImportError(notice) - else: - if use_dir_lock: - lock_class = MkdirLockFile - - elif lock_class is None: - lock_class = LockFile - self.directory = directory self.forever = forever self.filemode = filemode @@ -100,17 +90,17 @@ class _FileCacheMixin: self.lock_class = lock_class @staticmethod - def encode(x): + def encode(x: str) -> str: return hashlib.sha224(x.encode()).hexdigest() - def _fn(self, name): + def _fn(self, name: str) -> str: # NOTE: This method should not change as some may depend on it. # See: https://github.com/ionrock/cachecontrol/issues/63 hashed = self.encode(name) parts = list(hashed[:5]) + [hashed] return os.path.join(self.directory, *parts) - def get(self, key): + def get(self, key: str) -> Optional[bytes]: name = self._fn(key) try: with open(name, "rb") as fh: @@ -119,11 +109,13 @@ class _FileCacheMixin: except FileNotFoundError: return None - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + ) -> None: name = self._fn(key) self._write(name, value) - def _write(self, path, data: bytes): + def _write(self, path: str, data: bytes) -> None: """ Safely write the data to the given path. """ @@ -133,12 +125,12 @@ class _FileCacheMixin: except (IOError, OSError): pass - with self.lock_class(path) as lock: + with self.lock_class(path + ".lock"): # Write our actual file - with _secure_open_write(lock.path, self.filemode) as fh: + with _secure_open_write(path, self.filemode) as fh: fh.write(data) - def _delete(self, key, suffix): + def _delete(self, key: str, suffix: str) -> None: name = self._fn(key) + suffix if not self.forever: try: @@ -153,7 +145,7 @@ class FileCache(_FileCacheMixin, BaseCache): downloads. """ - def delete(self, key): + def delete(self, key: str) -> None: self._delete(key, "") @@ -163,23 +155,23 @@ class SeparateBodyFileCache(_FileCacheMixin, SeparateBodyBaseCache): peak memory usage. """ - def get_body(self, key): + def get_body(self, key: str) -> Optional["IO[bytes]"]: name = self._fn(key) + ".body" try: return open(name, "rb") except FileNotFoundError: return None - def set_body(self, key, body): + def set_body(self, key: str, body: bytes) -> None: name = self._fn(key) + ".body" self._write(name, body) - def delete(self, key): + def delete(self, key: str) -> None: self._delete(key, "") self._delete(key, ".body") -def url_to_file_path(url, filecache): +def url_to_file_path(url: str, filecache: FileCache) -> str: """Return the file cache path based on the URL. This does not ensure the file exists! diff --git a/src/pip/_vendor/cachecontrol/caches/redis_cache.py b/src/pip/_vendor/cachecontrol/caches/redis_cache.py index 2cba4b070..f7ae45d38 100644 --- a/src/pip/_vendor/cachecontrol/caches/redis_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/redis_cache.py @@ -4,36 +4,45 @@ from __future__ import division -from datetime import datetime +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Optional, Union + from pip._vendor.cachecontrol.cache import BaseCache +if TYPE_CHECKING: + from redis import Redis + class RedisCache(BaseCache): - - def __init__(self, conn): + def __init__(self, conn: "Redis[bytes]") -> None: self.conn = conn - def get(self, key): + def get(self, key: str) -> Optional[bytes]: return self.conn.get(key) - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: Optional[Union[int, datetime]] = None + ) -> None: if not expires: self.conn.set(key, value) elif isinstance(expires, datetime): - expires = expires - datetime.utcnow() - self.conn.setex(key, int(expires.total_seconds()), value) + now_utc = datetime.now(timezone.utc) + if expires.tzinfo is None: + now_utc = now_utc.replace(tzinfo=None) + delta = expires - now_utc + self.conn.setex(key, int(delta.total_seconds()), value) else: self.conn.setex(key, expires, value) - def delete(self, key): + def delete(self, key: str) -> None: self.conn.delete(key) - def clear(self): + def clear(self) -> None: """Helper for clearing all the keys in a database. Use with caution!""" for key in self.conn.keys(): self.conn.delete(key) - def close(self): + def close(self) -> None: """Redis uses connection pooling, no need to close the connection.""" pass diff --git a/src/pip/_vendor/cachecontrol/compat.py b/src/pip/_vendor/cachecontrol/compat.py deleted file mode 100644 index ccec9379d..000000000 --- a/src/pip/_vendor/cachecontrol/compat.py +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: 2015 Eric Larson -# -# SPDX-License-Identifier: Apache-2.0 - -try: - from urllib.parse import urljoin -except ImportError: - from urlparse import urljoin - - -try: - import cPickle as pickle -except ImportError: - import pickle - -# Handle the case where the requests module has been patched to not have -# urllib3 bundled as part of its source. -try: - from pip._vendor.requests.packages.urllib3.response import HTTPResponse -except ImportError: - from pip._vendor.urllib3.response import HTTPResponse - -try: - from pip._vendor.requests.packages.urllib3.util import is_fp_closed -except ImportError: - from pip._vendor.urllib3.util import is_fp_closed - -# Replicate some six behaviour -try: - text_type = unicode -except NameError: - text_type = str diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 7af0e002d..3365d9621 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -5,17 +5,25 @@ """ The httplib2 algorithms ported for use with requests. """ +import calendar import logging import re -import calendar import time from email.utils import parsedate_tz +from typing import TYPE_CHECKING, Collection, Dict, Mapping, Optional, Tuple, Union from pip._vendor.requests.structures import CaseInsensitiveDict -from .cache import DictCache, SeparateBodyBaseCache -from .serialize import Serializer +from pip._vendor.cachecontrol.cache import DictCache, SeparateBodyBaseCache +from pip._vendor.cachecontrol.serialize import Serializer +if TYPE_CHECKING: + from typing import Literal + + from pip._vendor.requests import PreparedRequest + from pip._vendor.urllib3 import HTTPResponse + + from pip._vendor.cachecontrol.cache import BaseCache logger = logging.getLogger(__name__) @@ -24,12 +32,14 @@ URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") PERMANENT_REDIRECT_STATUSES = (301, 308) -def parse_uri(uri): +def parse_uri(uri: str) -> Tuple[str, str, str, str, str]: """Parses a URI using the regex given in Appendix B of RFC 3986. (scheme, authority, path, query, fragment) = parse_uri(uri) """ - groups = URI.match(uri).groups() + match = URI.match(uri) + assert match is not None + groups = match.groups() return (groups[1], groups[3], groups[4], groups[6], groups[8]) @@ -37,7 +47,11 @@ class CacheController(object): """An interface to see if request should cached or not.""" def __init__( - self, cache=None, cache_etags=True, serializer=None, status_codes=None + self, + cache: Optional["BaseCache"] = None, + cache_etags: bool = True, + serializer: Optional[Serializer] = None, + status_codes: Optional[Collection[int]] = None, ): self.cache = DictCache() if cache is None else cache self.cache_etags = cache_etags @@ -45,7 +59,7 @@ class CacheController(object): self.cacheable_status_codes = status_codes or (200, 203, 300, 301, 308) @classmethod - def _urlnorm(cls, uri): + def _urlnorm(cls, uri: str) -> str: """Normalize the URL to create a safe key for the cache""" (scheme, authority, path, query, fragment) = parse_uri(uri) if not scheme or not authority: @@ -65,10 +79,12 @@ class CacheController(object): return defrag_uri @classmethod - def cache_url(cls, uri): + def cache_url(cls, uri: str) -> str: return cls._urlnorm(uri) - def parse_cache_control(self, headers): + def parse_cache_control( + self, headers: Mapping[str, str] + ) -> Dict[str, Optional[int]]: known_directives = { # https://tools.ietf.org/html/rfc7234#section-5.2 "max-age": (int, True), @@ -87,7 +103,7 @@ class CacheController(object): cc_headers = headers.get("cache-control", headers.get("Cache-Control", "")) - retval = {} + retval: Dict[str, Optional[int]] = {} for cc_directive in cc_headers.split(","): if not cc_directive.strip(): @@ -122,11 +138,12 @@ class CacheController(object): return retval - def _load_from_cache(self, request): + def _load_from_cache(self, request: "PreparedRequest") -> Optional["HTTPResponse"]: """ Load a cached response, or return None if it's not available. """ cache_url = request.url + assert cache_url is not None cache_data = self.cache.get(cache_url) if cache_data is None: logger.debug("No cache entry available") @@ -142,11 +159,14 @@ class CacheController(object): logger.warning("Cache entry deserialization failed, entry ignored") return result - def cached_request(self, request): + def cached_request( + self, request: "PreparedRequest" + ) -> Union["HTTPResponse", "Literal[False]"]: """ Return a cached response if it exists in the cache, otherwise return False. """ + assert request.url is not None cache_url = self.cache_url(request.url) logger.debug('Looking up "%s" in the cache', cache_url) cc = self.parse_cache_control(request.headers) @@ -182,7 +202,7 @@ class CacheController(object): logger.debug(msg) return resp - headers = CaseInsensitiveDict(resp.headers) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers) if not headers or "date" not in headers: if "etag" not in headers: # Without date or etag, the cached response can never be used @@ -193,7 +213,9 @@ class CacheController(object): return False now = time.time() - date = calendar.timegm(parsedate_tz(headers["date"])) + time_tuple = parsedate_tz(headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) current_age = max(0, now - date) logger.debug("Current age based on date: %i", current_age) @@ -207,28 +229,30 @@ class CacheController(object): freshness_lifetime = 0 # Check the max-age pragma in the cache control header - if "max-age" in resp_cc: - freshness_lifetime = resp_cc["max-age"] + max_age = resp_cc.get("max-age") + if max_age is not None: + freshness_lifetime = max_age logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime) # If there isn't a max-age, check for an expires header elif "expires" in headers: expires = parsedate_tz(headers["expires"]) if expires is not None: - expire_time = calendar.timegm(expires) - date + expire_time = calendar.timegm(expires[:6]) - date freshness_lifetime = max(0, expire_time) logger.debug("Freshness lifetime from expires: %i", freshness_lifetime) # Determine if we are setting freshness limit in the # request. Note, this overrides what was in the response. - if "max-age" in cc: - freshness_lifetime = cc["max-age"] + max_age = cc.get("max-age") + if max_age is not None: + freshness_lifetime = max_age logger.debug( "Freshness lifetime from request max-age: %i", freshness_lifetime ) - if "min-fresh" in cc: - min_fresh = cc["min-fresh"] + min_fresh = cc.get("min-fresh") + if min_fresh is not None: # adjust our current age by our min fresh current_age += min_fresh logger.debug("Adjusted current age from min-fresh: %i", current_age) @@ -247,12 +271,12 @@ class CacheController(object): # return the original handler return False - def conditional_headers(self, request): + def conditional_headers(self, request: "PreparedRequest") -> Dict[str, str]: resp = self._load_from_cache(request) new_headers = {} if resp: - headers = CaseInsensitiveDict(resp.headers) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers) if "etag" in headers: new_headers["If-None-Match"] = headers["ETag"] @@ -262,7 +286,14 @@ class CacheController(object): return new_headers - def _cache_set(self, cache_url, request, response, body=None, expires_time=None): + def _cache_set( + self, + cache_url: str, + request: "PreparedRequest", + response: "HTTPResponse", + body: Optional[bytes] = None, + expires_time: Optional[int] = None, + ) -> None: """ Store the data in the cache. """ @@ -285,7 +316,13 @@ class CacheController(object): expires=expires_time, ) - def cache_response(self, request, response, body=None, status_codes=None): + def cache_response( + self, + request: "PreparedRequest", + response: "HTTPResponse", + body: Optional[bytes] = None, + status_codes: Optional[Collection[int]] = None, + ) -> None: """ Algorithm for caching requests. @@ -300,10 +337,14 @@ class CacheController(object): ) return - response_headers = CaseInsensitiveDict(response.headers) + response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + response.headers + ) if "date" in response_headers: - date = calendar.timegm(parsedate_tz(response_headers["date"])) + time_tuple = parsedate_tz(response_headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) else: date = 0 @@ -322,6 +363,7 @@ class CacheController(object): cc_req = self.parse_cache_control(request.headers) cc = self.parse_cache_control(response_headers) + assert request.url is not None cache_url = self.cache_url(request.url) logger.debug('Updating cache with response from "%s"', cache_url) @@ -354,7 +396,7 @@ class CacheController(object): if response_headers.get("expires"): expires = parsedate_tz(response_headers["expires"]) if expires is not None: - expires_time = calendar.timegm(expires) - date + expires_time = calendar.timegm(expires[:6]) - date expires_time = max(expires_time, 14 * 86400) @@ -372,11 +414,14 @@ class CacheController(object): # is no date header then we can't do anything about expiring # the cache. elif "date" in response_headers: - date = calendar.timegm(parsedate_tz(response_headers["date"])) + time_tuple = parsedate_tz(response_headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) # cache when there is a max-age > 0 - if "max-age" in cc and cc["max-age"] > 0: + max_age = cc.get("max-age") + if max_age is not None and max_age > 0: logger.debug("Caching b/c date exists and max-age > 0") - expires_time = cc["max-age"] + expires_time = max_age self._cache_set( cache_url, request, @@ -391,7 +436,7 @@ class CacheController(object): if response_headers["expires"]: expires = parsedate_tz(response_headers["expires"]) if expires is not None: - expires_time = calendar.timegm(expires) - date + expires_time = calendar.timegm(expires[:6]) - date else: expires_time = None @@ -408,13 +453,16 @@ class CacheController(object): expires_time, ) - def update_cached_response(self, request, response): + def update_cached_response( + self, request: "PreparedRequest", response: "HTTPResponse" + ) -> "HTTPResponse": """On a 304 we will get a new set of headers that we want to update our cached value with, assuming we have one. This should only ever be called when we've sent an ETag and gotten a 304 as the response. """ + assert request.url is not None cache_url = self.cache_url(request.url) cached_response = self._load_from_cache(request) @@ -434,7 +482,7 @@ class CacheController(object): cached_response.headers.update( dict( (k, v) - for k, v in response.headers.items() + for k, v in response.headers.items() # type: ignore[no-untyped-call] if k.lower() not in excluded_headers ) ) diff --git a/src/pip/_vendor/cachecontrol/filewrapper.py b/src/pip/_vendor/cachecontrol/filewrapper.py index f5ed5f6f6..472ba6001 100644 --- a/src/pip/_vendor/cachecontrol/filewrapper.py +++ b/src/pip/_vendor/cachecontrol/filewrapper.py @@ -2,8 +2,12 @@ # # SPDX-License-Identifier: Apache-2.0 -from tempfile import NamedTemporaryFile import mmap +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING, Any, Callable, Optional + +if TYPE_CHECKING: + from http.client import HTTPResponse class CallbackFileWrapper(object): @@ -25,12 +29,14 @@ class CallbackFileWrapper(object): performance impact. """ - def __init__(self, fp, callback): + def __init__( + self, fp: "HTTPResponse", callback: Optional[Callable[[bytes], None]] + ) -> None: self.__buf = NamedTemporaryFile("rb+", delete=True) self.__fp = fp self.__callback = callback - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: # The vaguaries of garbage collection means that self.__fp is # not always set. By using __getattribute__ and the private # name[0] allows looking up the attribute value and raising an @@ -42,7 +48,7 @@ class CallbackFileWrapper(object): fp = self.__getattribute__("_CallbackFileWrapper__fp") return getattr(fp, name) - def __is_fp_closed(self): + def __is_fp_closed(self) -> bool: try: return self.__fp.fp is None @@ -50,7 +56,8 @@ class CallbackFileWrapper(object): pass try: - return self.__fp.closed + closed: bool = self.__fp.closed + return closed except AttributeError: pass @@ -59,7 +66,7 @@ class CallbackFileWrapper(object): # TODO: Add some logging here... return False - def _close(self): + def _close(self) -> None: if self.__callback: if self.__buf.tell() == 0: # Empty file: @@ -86,8 +93,8 @@ class CallbackFileWrapper(object): # Important when caching big files. self.__buf.close() - def read(self, amt=None): - data = self.__fp.read(amt) + def read(self, amt: Optional[int] = None) -> bytes: + data: bytes = self.__fp.read(amt) if data: # We may be dealing with b'', a sign that things are over: # it's passed e.g. after we've already closed self.__buf. @@ -97,8 +104,8 @@ class CallbackFileWrapper(object): return data - def _safe_read(self, amt): - data = self.__fp._safe_read(amt) + def _safe_read(self, amt: int) -> bytes: + data: bytes = self.__fp._safe_read(amt) # type: ignore[attr-defined] if amt == 2 and data == b"\r\n": # urllib executes this read to toss the CRLF at the end # of the chunk. diff --git a/src/pip/_vendor/cachecontrol/heuristics.py b/src/pip/_vendor/cachecontrol/heuristics.py index ebe4a96f5..1e88ada68 100644 --- a/src/pip/_vendor/cachecontrol/heuristics.py +++ b/src/pip/_vendor/cachecontrol/heuristics.py @@ -4,26 +4,27 @@ import calendar import time - +from datetime import datetime, timedelta, timezone from email.utils import formatdate, parsedate, parsedate_tz +from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional -from datetime import datetime, timedelta +if TYPE_CHECKING: + from pip._vendor.urllib3 import HTTPResponse TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT" -def expire_after(delta, date=None): - date = date or datetime.utcnow() +def expire_after(delta: timedelta, date: Optional[datetime] = None) -> datetime: + date = date or datetime.now(timezone.utc) return date + delta -def datetime_to_header(dt): +def datetime_to_header(dt: datetime) -> str: return formatdate(calendar.timegm(dt.timetuple())) class BaseHeuristic(object): - - def warning(self, response): + def warning(self, response: "HTTPResponse") -> Optional[str]: """ Return a valid 1xx warning header value describing the cache adjustments. @@ -34,7 +35,7 @@ class BaseHeuristic(object): """ return '110 - "Response is Stale"' - def update_headers(self, response): + def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: """Update the response headers with any new headers. NOTE: This SHOULD always include some Warning header to @@ -43,7 +44,7 @@ class BaseHeuristic(object): """ return {} - def apply(self, response): + def apply(self, response: "HTTPResponse") -> "HTTPResponse": updated_headers = self.update_headers(response) if updated_headers: @@ -61,12 +62,12 @@ class OneDayCache(BaseHeuristic): future. """ - def update_headers(self, response): + def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: headers = {} if "expires" not in response.headers: date = parsedate(response.headers["date"]) - expires = expire_after(timedelta(days=1), date=datetime(*date[:6])) + expires = expire_after(timedelta(days=1), date=datetime(*date[:6], tzinfo=timezone.utc)) # type: ignore[misc] headers["expires"] = datetime_to_header(expires) headers["cache-control"] = "public" return headers @@ -77,14 +78,14 @@ class ExpiresAfter(BaseHeuristic): Cache **all** requests for a defined time period. """ - def __init__(self, **kw): + def __init__(self, **kw: Any) -> None: self.delta = timedelta(**kw) - def update_headers(self, response): + def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: expires = expire_after(self.delta) return {"expires": datetime_to_header(expires), "cache-control": "public"} - def warning(self, response): + def warning(self, response: "HTTPResponse") -> Optional[str]: tmpl = "110 - Automatically cached for %s. Response might be stale" return tmpl % self.delta @@ -101,12 +102,23 @@ class LastModified(BaseHeuristic): http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397 Unlike mozilla we limit this to 24-hr. """ + cacheable_by_default_statuses = { - 200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501 + 200, + 203, + 204, + 206, + 300, + 301, + 404, + 405, + 410, + 414, + 501, } - def update_headers(self, resp): - headers = resp.headers + def update_headers(self, resp: "HTTPResponse") -> Dict[str, str]: + headers: Mapping[str, str] = resp.headers if "expires" in headers: return {} @@ -120,9 +132,11 @@ class LastModified(BaseHeuristic): if "date" not in headers or "last-modified" not in headers: return {} - date = calendar.timegm(parsedate_tz(headers["date"])) + time_tuple = parsedate_tz(headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) last_modified = parsedate(headers["last-modified"]) - if date is None or last_modified is None: + if last_modified is None: return {} now = time.time() @@ -135,5 +149,5 @@ class LastModified(BaseHeuristic): expires = date + freshness_lifetime return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))} - def warning(self, resp): + def warning(self, resp: "HTTPResponse") -> Optional[str]: return None diff --git a/src/pip/_vendor/cachecontrol/py.typed b/src/pip/_vendor/cachecontrol/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/pip/_vendor/cachecontrol/serialize.py b/src/pip/_vendor/cachecontrol/serialize.py index 7fe1a3e33..f21eaea6f 100644 --- a/src/pip/_vendor/cachecontrol/serialize.py +++ b/src/pip/_vendor/cachecontrol/serialize.py @@ -5,19 +5,23 @@ import base64 import io import json +import pickle import zlib +from typing import IO, TYPE_CHECKING, Any, Mapping, Optional from pip._vendor import msgpack from pip._vendor.requests.structures import CaseInsensitiveDict +from pip._vendor.urllib3 import HTTPResponse -from .compat import HTTPResponse, pickle, text_type +if TYPE_CHECKING: + from pip._vendor.requests import PreparedRequest, Request -def _b64_decode_bytes(b): +def _b64_decode_bytes(b: str) -> bytes: return base64.b64decode(b.encode("ascii")) -def _b64_decode_str(s): +def _b64_decode_str(s: str) -> str: return _b64_decode_bytes(s).decode("utf8") @@ -25,54 +29,57 @@ _default_body_read = object() class Serializer(object): - def dumps(self, request, response, body=None): - response_headers = CaseInsensitiveDict(response.headers) + def dumps( + self, + request: "PreparedRequest", + response: HTTPResponse, + body: Optional[bytes] = None, + ) -> bytes: + response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + response.headers + ) if body is None: # When a body isn't passed in, we'll read the response. We # also update the response with a new file handler to be # sure it acts as though it was never read. body = response.read(decode_content=False) - response._fp = io.BytesIO(body) + response._fp = io.BytesIO(body) # type: ignore[attr-defined] + response.length_remaining = len(body) - # NOTE: This is all a bit weird, but it's really important that on - # Python 2.x these objects are unicode and not str, even when - # they contain only ascii. The problem here is that msgpack - # understands the difference between unicode and bytes and we - # have it set to differentiate between them, however Python 2 - # doesn't know the difference. Forcing these to unicode will be - # enough to have msgpack know the difference. data = { - u"response": { - u"body": body, # Empty bytestring if body is stored separately - u"headers": dict( - (text_type(k), text_type(v)) for k, v in response.headers.items() - ), - u"status": response.status, - u"version": response.version, - u"reason": text_type(response.reason), - u"strict": response.strict, - u"decode_content": response.decode_content, + "response": { + "body": body, # Empty bytestring if body is stored separately + "headers": dict((str(k), str(v)) for k, v in response.headers.items()), # type: ignore[no-untyped-call] + "status": response.status, + "version": response.version, + "reason": str(response.reason), + "decode_content": response.decode_content, } } # Construct our vary headers - data[u"vary"] = {} - if u"vary" in response_headers: - varied_headers = response_headers[u"vary"].split(",") + data["vary"] = {} + if "vary" in response_headers: + varied_headers = response_headers["vary"].split(",") for header in varied_headers: - header = text_type(header).strip() + header = str(header).strip() header_value = request.headers.get(header, None) if header_value is not None: - header_value = text_type(header_value) - data[u"vary"][header] = header_value + header_value = str(header_value) + data["vary"][header] = header_value return b",".join([b"cc=4", msgpack.dumps(data, use_bin_type=True)]) - def loads(self, request, data, body_file=None): + def loads( + self, + request: "PreparedRequest", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: # Short circuit if we've been given an empty set of data if not data: - return + return None # Determine what version of the serializer the data was serialized # with @@ -88,18 +95,23 @@ class Serializer(object): ver = b"cc=0" # Get the version number out of the cc=N - ver = ver.split(b"=", 1)[-1].decode("ascii") + verstr = ver.split(b"=", 1)[-1].decode("ascii") # Dispatch to the actual load method for the given version try: - return getattr(self, "_loads_v{}".format(ver))(request, data, body_file) + return getattr(self, "_loads_v{}".format(verstr))(request, data, body_file) # type: ignore[no-any-return] except AttributeError: # This is a version we don't have a loads function for, so we'll # just treat it as a miss and return None - return + return None - def prepare_response(self, request, cached, body_file=None): + def prepare_response( + self, + request: "Request", + cached: Mapping[str, Any], + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: """Verify our vary headers match and construct a real urllib3 HTTPResponse object. """ @@ -108,23 +120,26 @@ class Serializer(object): # This case is also handled in the controller code when creating # a cache entry, but is left here for backwards compatibility. if "*" in cached.get("vary", {}): - return + return None # Ensure that the Vary headers for the cached response match our # request for header, value in cached.get("vary", {}).items(): if request.headers.get(header, None) != value: - return + return None body_raw = cached["response"].pop("body") - headers = CaseInsensitiveDict(data=cached["response"]["headers"]) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + data=cached["response"]["headers"] + ) if headers.get("transfer-encoding", "") == "chunked": headers.pop("transfer-encoding") cached["response"]["headers"] = headers try: + body: "IO[bytes]" if body_file is None: body = io.BytesIO(body_raw) else: @@ -138,28 +153,46 @@ class Serializer(object): # TypeError: 'str' does not support the buffer interface body = io.BytesIO(body_raw.encode("utf8")) + # Discard any `strict` parameter serialized by older version of cachecontrol. + cached["response"].pop("strict", None) + return HTTPResponse(body=body, preload_content=False, **cached["response"]) - def _loads_v0(self, request, data, body_file=None): + def _loads_v0( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> None: # The original legacy cache data. This doesn't contain enough # information to construct everything we need, so we'll treat this as # a miss. return - def _loads_v1(self, request, data, body_file=None): + def _loads_v1( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: try: cached = pickle.loads(data) except ValueError: - return + return None return self.prepare_response(request, cached, body_file) - def _loads_v2(self, request, data, body_file=None): + def _loads_v2( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: assert body_file is None try: cached = json.loads(zlib.decompress(data).decode("utf8")) except (ValueError, zlib.error): - return + return None # We need to decode the items that we've base64 encoded cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"]) @@ -175,16 +208,26 @@ class Serializer(object): return self.prepare_response(request, cached, body_file) - def _loads_v3(self, request, data, body_file): + def _loads_v3( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> None: # Due to Python 2 encoding issues, it's impossible to know for sure # exactly how to load v3 entries, thus we'll treat these as a miss so # that they get rewritten out as v4 entries. return - def _loads_v4(self, request, data, body_file=None): + def _loads_v4( + self, + request: "Request", + data: bytes, + body_file: Optional["IO[bytes]"] = None, + ) -> Optional[HTTPResponse]: try: cached = msgpack.loads(data, raw=False) except ValueError: - return + return None return self.prepare_response(request, cached, body_file) diff --git a/src/pip/_vendor/cachecontrol/wrapper.py b/src/pip/_vendor/cachecontrol/wrapper.py index b6ee7f203..293e69fe7 100644 --- a/src/pip/_vendor/cachecontrol/wrapper.py +++ b/src/pip/_vendor/cachecontrol/wrapper.py @@ -2,21 +2,30 @@ # # SPDX-License-Identifier: Apache-2.0 -from .adapter import CacheControlAdapter -from .cache import DictCache +from typing import TYPE_CHECKING, Collection, Optional, Type + +from pip._vendor.cachecontrol.adapter import CacheControlAdapter +from pip._vendor.cachecontrol.cache import DictCache + +if TYPE_CHECKING: + from pip._vendor import requests + + from pip._vendor.cachecontrol.cache import BaseCache + from pip._vendor.cachecontrol.controller import CacheController + from pip._vendor.cachecontrol.heuristics import BaseHeuristic + from pip._vendor.cachecontrol.serialize import Serializer def CacheControl( - sess, - cache=None, - cache_etags=True, - serializer=None, - heuristic=None, - controller_class=None, - adapter_class=None, - cacheable_methods=None, -): - + sess: "requests.Session", + cache: Optional["BaseCache"] = None, + cache_etags: bool = True, + serializer: Optional["Serializer"] = None, + heuristic: Optional["BaseHeuristic"] = None, + controller_class: Optional[Type["CacheController"]] = None, + adapter_class: Optional[Type[CacheControlAdapter]] = None, + cacheable_methods: Optional[Collection[str]] = None, +) -> "requests.Session": cache = DictCache() if cache is None else cache adapter_class = adapter_class or CacheControlAdapter adapter = adapter_class( diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index dcf89dc04..d0f4c71cc 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -1,4 +1,4 @@ -CacheControl==0.12.11 # Make sure to update the license in pyproject.toml for this. +CacheControl==0.13.0 # Make sure to update the license in pyproject.toml for this. colorama==0.4.6 distlib==0.3.6 distro==1.8.0 From 9fb93c478ef7d5e1423cc66467bb63c686864828 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Jun 2023 14:00:15 -0400 Subject: [PATCH 077/216] mypy fix. --- src/pip/_internal/network/cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index d6b8ccdcf..a4d136205 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -3,7 +3,8 @@ import os from contextlib import contextmanager -from typing import BinaryIO, Generator, Optional +from datetime import datetime +from typing import BinaryIO, Generator, Optional, Union from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache from pip._vendor.cachecontrol.caches import SeparateBodyFileCache @@ -62,7 +63,9 @@ class SafeFileCache(SeparateBodyBaseCache): replace(f.name, path) - def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None: + def set( + self, key: str, value: bytes, expires: Union[int, datetime, None] = None + ) -> None: path = self._get_cache_path(key) self._write(path, value) From 28590a0a0809b3bb8999b4d08aa93bd9ffb3458d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 09:29:35 -0400 Subject: [PATCH 078/216] Improve documentation of caching and the cache subcommand. --- docs/html/topics/caching.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/html/topics/caching.md b/docs/html/topics/caching.md index 954cebe40..19bd064a7 100644 --- a/docs/html/topics/caching.md +++ b/docs/html/topics/caching.md @@ -27,6 +27,12 @@ While this cache attempts to minimize network activity, it does not prevent network access altogether. If you want a local install solution that circumvents accessing PyPI, see {ref}`Installing from local packages`. +In versions prior to 23.2, this cache was stored in a directory called `http` in +the main cache directory (see below for its location). In 23.2 and later, a new +cache format is used, stored in a directory called `http-v2`. If you have +completely switched to newer versions of `pip`, you may wish to delete the old +directory. + (wheel-caching)= ### Locally built wheels @@ -124,11 +130,11 @@ The {ref}`pip cache` command can be used to manage pip's cache. ### Removing a single package -`pip cache remove setuptools` removes all wheel files related to setuptools from pip's cache. +`pip cache remove setuptools` removes all wheel files related to setuptools from pip's cache. HTTP cache files are not removed at this time. ### Removing the cache -`pip cache purge` will clear all wheel files from pip's cache. +`pip cache purge` will clear all files from pip's wheel and HTTP caches. ### Listing cached files From dcd2d5e344f27149789f05edb9da45994eac2473 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 09:30:30 -0400 Subject: [PATCH 079/216] Update CacheControl to 0.13.1. --- src/pip/_vendor/cachecontrol/__init__.py | 2 +- src/pip/_vendor/cachecontrol/_cmd.py | 5 +- src/pip/_vendor/cachecontrol/adapter.py | 51 ++++---- src/pip/_vendor/cachecontrol/cache.py | 18 +-- .../_vendor/cachecontrol/caches/file_cache.py | 17 +-- .../cachecontrol/caches/redis_cache.py | 10 +- src/pip/_vendor/cachecontrol/controller.py | 58 +++++---- src/pip/_vendor/cachecontrol/filewrapper.py | 9 +- src/pip/_vendor/cachecontrol/heuristics.py | 23 ++-- src/pip/_vendor/cachecontrol/serialize.py | 111 +++++++----------- src/pip/_vendor/cachecontrol/wrapper.py | 19 +-- src/pip/_vendor/vendor.txt | 2 +- 12 files changed, 149 insertions(+), 176 deletions(-) diff --git a/src/pip/_vendor/cachecontrol/__init__.py b/src/pip/_vendor/cachecontrol/__init__.py index 3701cdd6b..4d20bc9b1 100644 --- a/src/pip/_vendor/cachecontrol/__init__.py +++ b/src/pip/_vendor/cachecontrol/__init__.py @@ -8,7 +8,7 @@ Make it easy to import from cachecontrol without long namespaces. """ __author__ = "Eric Larson" __email__ = "eric@ionrock.org" -__version__ = "0.13.0" +__version__ = "0.13.1" from pip._vendor.cachecontrol.adapter import CacheControlAdapter from pip._vendor.cachecontrol.controller import CacheController diff --git a/src/pip/_vendor/cachecontrol/_cmd.py b/src/pip/_vendor/cachecontrol/_cmd.py index ab4dac3dd..2c84208a5 100644 --- a/src/pip/_vendor/cachecontrol/_cmd.py +++ b/src/pip/_vendor/cachecontrol/_cmd.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import logging from argparse import ArgumentParser @@ -36,7 +37,7 @@ def get_session() -> requests.Session: return sess -def get_args() -> "Namespace": +def get_args() -> Namespace: parser = ArgumentParser() parser.add_argument("url", help="The URL to try and cache") return parser.parse_args() @@ -53,7 +54,7 @@ def main() -> None: setup_logging() # try setting the cache - cache_controller: "CacheController" = ( + cache_controller: CacheController = ( sess.cache_controller # type: ignore[attr-defined] ) cache_controller.cache_response(resp.request, resp.raw) diff --git a/src/pip/_vendor/cachecontrol/adapter.py b/src/pip/_vendor/cachecontrol/adapter.py index 83c08e003..3e83e308d 100644 --- a/src/pip/_vendor/cachecontrol/adapter.py +++ b/src/pip/_vendor/cachecontrol/adapter.py @@ -1,11 +1,12 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import functools import types import zlib -from typing import TYPE_CHECKING, Any, Collection, Mapping, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Collection, Mapping from pip._vendor.requests.adapters import HTTPAdapter @@ -27,16 +28,16 @@ class CacheControlAdapter(HTTPAdapter): def __init__( self, - cache: Optional["BaseCache"] = None, + cache: BaseCache | None = None, cache_etags: bool = True, - controller_class: Optional[Type[CacheController]] = None, - serializer: Optional["Serializer"] = None, - heuristic: Optional["BaseHeuristic"] = None, - cacheable_methods: Optional[Collection[str]] = None, + controller_class: type[CacheController] | None = None, + serializer: Serializer | None = None, + heuristic: BaseHeuristic | None = None, + cacheable_methods: Collection[str] | None = None, *args: Any, **kw: Any, ) -> None: - super(CacheControlAdapter, self).__init__(*args, **kw) + super().__init__(*args, **kw) self.cache = DictCache() if cache is None else cache self.heuristic = heuristic self.cacheable_methods = cacheable_methods or ("GET",) @@ -48,16 +49,14 @@ class CacheControlAdapter(HTTPAdapter): def send( self, - request: "PreparedRequest", + request: PreparedRequest, stream: bool = False, - timeout: Union[None, float, Tuple[float, float], Tuple[float, None]] = None, - verify: Union[bool, str] = True, - cert: Union[ - None, bytes, str, Tuple[Union[bytes, str], Union[bytes, str]] - ] = None, - proxies: Optional[Mapping[str, str]] = None, - cacheable_methods: Optional[Collection[str]] = None, - ) -> "Response": + timeout: None | float | tuple[float, float] | tuple[float, None] = None, + verify: bool | str = True, + cert: (None | bytes | str | tuple[bytes | str, bytes | str]) = None, + proxies: Mapping[str, str] | None = None, + cacheable_methods: Collection[str] | None = None, + ) -> Response: """ Send a request. Use the request information to see if it exists in the cache and cache the response if we need to and can. @@ -74,19 +73,17 @@ class CacheControlAdapter(HTTPAdapter): # check for etags and add headers if appropriate request.headers.update(self.controller.conditional_headers(request)) - resp = super(CacheControlAdapter, self).send( - request, stream, timeout, verify, cert, proxies - ) + resp = super().send(request, stream, timeout, verify, cert, proxies) return resp def build_response( self, - request: "PreparedRequest", - response: "HTTPResponse", + request: PreparedRequest, + response: HTTPResponse, from_cache: bool = False, - cacheable_methods: Optional[Collection[str]] = None, - ) -> "Response": + cacheable_methods: Collection[str] | None = None, + ) -> Response: """ Build a response by making a request or using the cache. @@ -137,7 +134,7 @@ class CacheControlAdapter(HTTPAdapter): if response.chunked: super_update_chunk_length = response._update_chunk_length # type: ignore[attr-defined] - def _update_chunk_length(self: "HTTPResponse") -> None: + def _update_chunk_length(self: HTTPResponse) -> None: super_update_chunk_length() if self.chunk_left == 0: self._fp._close() # type: ignore[attr-defined] @@ -146,9 +143,7 @@ class CacheControlAdapter(HTTPAdapter): _update_chunk_length, response ) - resp: "Response" = super( # type: ignore[no-untyped-call] - CacheControlAdapter, self - ).build_response(request, response) + resp: Response = super().build_response(request, response) # type: ignore[no-untyped-call] # See if we should invalidate the cache. if request.method in self.invalidating_methods and resp.ok: @@ -163,4 +158,4 @@ class CacheControlAdapter(HTTPAdapter): def close(self) -> None: self.cache.close() - super(CacheControlAdapter, self).close() # type: ignore[no-untyped-call] + super().close() # type: ignore[no-untyped-call] diff --git a/src/pip/_vendor/cachecontrol/cache.py b/src/pip/_vendor/cachecontrol/cache.py index 61031d234..3293b0057 100644 --- a/src/pip/_vendor/cachecontrol/cache.py +++ b/src/pip/_vendor/cachecontrol/cache.py @@ -6,19 +6,21 @@ The cache object API for implementing caches. The default is a thread safe in-memory dictionary. """ +from __future__ import annotations + from threading import Lock -from typing import IO, TYPE_CHECKING, MutableMapping, Optional, Union +from typing import IO, TYPE_CHECKING, MutableMapping if TYPE_CHECKING: from datetime import datetime -class BaseCache(object): - def get(self, key: str) -> Optional[bytes]: +class BaseCache: + def get(self, key: str) -> bytes | None: raise NotImplementedError() def set( - self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + self, key: str, value: bytes, expires: int | datetime | None = None ) -> None: raise NotImplementedError() @@ -30,15 +32,15 @@ class BaseCache(object): class DictCache(BaseCache): - def __init__(self, init_dict: Optional[MutableMapping[str, bytes]] = None) -> None: + def __init__(self, init_dict: MutableMapping[str, bytes] | None = None) -> None: self.lock = Lock() self.data = init_dict or {} - def get(self, key: str) -> Optional[bytes]: + def get(self, key: str) -> bytes | None: return self.data.get(key, None) def set( - self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + self, key: str, value: bytes, expires: int | datetime | None = None ) -> None: with self.lock: self.data.update({key: value}) @@ -65,7 +67,7 @@ class SeparateBodyBaseCache(BaseCache): def set_body(self, key: str, body: bytes) -> None: raise NotImplementedError() - def get_body(self, key: str) -> Optional["IO[bytes]"]: + def get_body(self, key: str) -> IO[bytes] | None: """ Return the body as file-like object. """ diff --git a/src/pip/_vendor/cachecontrol/caches/file_cache.py b/src/pip/_vendor/cachecontrol/caches/file_cache.py index 0437c4e8a..1fd280130 100644 --- a/src/pip/_vendor/cachecontrol/caches/file_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/file_cache.py @@ -1,11 +1,12 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import hashlib import os from textwrap import dedent -from typing import IO, TYPE_CHECKING, Optional, Type, Union +from typing import IO, TYPE_CHECKING from pip._vendor.cachecontrol.cache import BaseCache, SeparateBodyBaseCache from pip._vendor.cachecontrol.controller import CacheController @@ -16,7 +17,7 @@ if TYPE_CHECKING: from filelock import BaseFileLock -def _secure_open_write(filename: str, fmode: int) -> "IO[bytes]": +def _secure_open_write(filename: str, fmode: int) -> IO[bytes]: # We only want to write to this file, so open it in write only mode flags = os.O_WRONLY @@ -39,7 +40,7 @@ def _secure_open_write(filename: str, fmode: int) -> "IO[bytes]": # there try: os.remove(filename) - except (IOError, OSError): + except OSError: # The file must not exist already, so we can just skip ahead to opening pass @@ -66,7 +67,7 @@ class _FileCacheMixin: forever: bool = False, filemode: int = 0o0600, dirmode: int = 0o0700, - lock_class: Optional[Type["BaseFileLock"]] = None, + lock_class: type[BaseFileLock] | None = None, ) -> None: try: if lock_class is None: @@ -100,7 +101,7 @@ class _FileCacheMixin: parts = list(hashed[:5]) + [hashed] return os.path.join(self.directory, *parts) - def get(self, key: str) -> Optional[bytes]: + def get(self, key: str) -> bytes | None: name = self._fn(key) try: with open(name, "rb") as fh: @@ -110,7 +111,7 @@ class _FileCacheMixin: return None def set( - self, key: str, value: bytes, expires: Optional[Union[int, "datetime"]] = None + self, key: str, value: bytes, expires: int | datetime | None = None ) -> None: name = self._fn(key) self._write(name, value) @@ -122,7 +123,7 @@ class _FileCacheMixin: # Make sure the directory exists try: os.makedirs(os.path.dirname(path), self.dirmode) - except (IOError, OSError): + except OSError: pass with self.lock_class(path + ".lock"): @@ -155,7 +156,7 @@ class SeparateBodyFileCache(_FileCacheMixin, SeparateBodyBaseCache): peak memory usage. """ - def get_body(self, key: str) -> Optional["IO[bytes]"]: + def get_body(self, key: str) -> IO[bytes] | None: name = self._fn(key) + ".body" try: return open(name, "rb") diff --git a/src/pip/_vendor/cachecontrol/caches/redis_cache.py b/src/pip/_vendor/cachecontrol/caches/redis_cache.py index f7ae45d38..f4f68c47b 100644 --- a/src/pip/_vendor/cachecontrol/caches/redis_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/redis_cache.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -from __future__ import division from datetime import datetime, timezone -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from pip._vendor.cachecontrol.cache import BaseCache @@ -14,14 +14,14 @@ if TYPE_CHECKING: class RedisCache(BaseCache): - def __init__(self, conn: "Redis[bytes]") -> None: + def __init__(self, conn: Redis[bytes]) -> None: self.conn = conn - def get(self, key: str) -> Optional[bytes]: + def get(self, key: str) -> bytes | None: return self.conn.get(key) def set( - self, key: str, value: bytes, expires: Optional[Union[int, datetime]] = None + self, key: str, value: bytes, expires: int | datetime | None = None ) -> None: if not expires: self.conn.set(key, value) diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 3365d9621..586b9f97b 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -5,12 +5,14 @@ """ The httplib2 algorithms ported for use with requests. """ +from __future__ import annotations + import calendar import logging import re import time from email.utils import parsedate_tz -from typing import TYPE_CHECKING, Collection, Dict, Mapping, Optional, Tuple, Union +from typing import TYPE_CHECKING, Collection, Mapping from pip._vendor.requests.structures import CaseInsensitiveDict @@ -32,7 +34,7 @@ URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") PERMANENT_REDIRECT_STATUSES = (301, 308) -def parse_uri(uri: str) -> Tuple[str, str, str, str, str]: +def parse_uri(uri: str) -> tuple[str, str, str, str, str]: """Parses a URI using the regex given in Appendix B of RFC 3986. (scheme, authority, path, query, fragment) = parse_uri(uri) @@ -43,15 +45,15 @@ def parse_uri(uri: str) -> Tuple[str, str, str, str, str]: return (groups[1], groups[3], groups[4], groups[6], groups[8]) -class CacheController(object): +class CacheController: """An interface to see if request should cached or not.""" def __init__( self, - cache: Optional["BaseCache"] = None, + cache: BaseCache | None = None, cache_etags: bool = True, - serializer: Optional[Serializer] = None, - status_codes: Optional[Collection[int]] = None, + serializer: Serializer | None = None, + status_codes: Collection[int] | None = None, ): self.cache = DictCache() if cache is None else cache self.cache_etags = cache_etags @@ -82,9 +84,7 @@ class CacheController(object): def cache_url(cls, uri: str) -> str: return cls._urlnorm(uri) - def parse_cache_control( - self, headers: Mapping[str, str] - ) -> Dict[str, Optional[int]]: + def parse_cache_control(self, headers: Mapping[str, str]) -> dict[str, int | None]: known_directives = { # https://tools.ietf.org/html/rfc7234#section-5.2 "max-age": (int, True), @@ -103,7 +103,7 @@ class CacheController(object): cc_headers = headers.get("cache-control", headers.get("Cache-Control", "")) - retval: Dict[str, Optional[int]] = {} + retval: dict[str, int | None] = {} for cc_directive in cc_headers.split(","): if not cc_directive.strip(): @@ -138,7 +138,7 @@ class CacheController(object): return retval - def _load_from_cache(self, request: "PreparedRequest") -> Optional["HTTPResponse"]: + def _load_from_cache(self, request: PreparedRequest) -> HTTPResponse | None: """ Load a cached response, or return None if it's not available. """ @@ -159,9 +159,7 @@ class CacheController(object): logger.warning("Cache entry deserialization failed, entry ignored") return result - def cached_request( - self, request: "PreparedRequest" - ) -> Union["HTTPResponse", "Literal[False]"]: + def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[False]: """ Return a cached response if it exists in the cache, otherwise return False. @@ -271,7 +269,7 @@ class CacheController(object): # return the original handler return False - def conditional_headers(self, request: "PreparedRequest") -> Dict[str, str]: + def conditional_headers(self, request: PreparedRequest) -> dict[str, str]: resp = self._load_from_cache(request) new_headers = {} @@ -289,10 +287,10 @@ class CacheController(object): def _cache_set( self, cache_url: str, - request: "PreparedRequest", - response: "HTTPResponse", - body: Optional[bytes] = None, - expires_time: Optional[int] = None, + request: PreparedRequest, + response: HTTPResponse, + body: bytes | None = None, + expires_time: int | None = None, ) -> None: """ Store the data in the cache. @@ -318,10 +316,10 @@ class CacheController(object): def cache_response( self, - request: "PreparedRequest", - response: "HTTPResponse", - body: Optional[bytes] = None, - status_codes: Optional[Collection[int]] = None, + request: PreparedRequest, + response: HTTPResponse, + body: bytes | None = None, + status_codes: Collection[int] | None = None, ) -> None: """ Algorithm for caching requests. @@ -400,7 +398,7 @@ class CacheController(object): expires_time = max(expires_time, 14 * 86400) - logger.debug("etag object cached for {0} seconds".format(expires_time)) + logger.debug(f"etag object cached for {expires_time} seconds") logger.debug("Caching due to etag") self._cache_set(cache_url, request, response, body, expires_time) @@ -441,7 +439,7 @@ class CacheController(object): expires_time = None logger.debug( - "Caching b/c of expires header. expires in {0} seconds".format( + "Caching b/c of expires header. expires in {} seconds".format( expires_time ) ) @@ -454,8 +452,8 @@ class CacheController(object): ) def update_cached_response( - self, request: "PreparedRequest", response: "HTTPResponse" - ) -> "HTTPResponse": + self, request: PreparedRequest, response: HTTPResponse + ) -> HTTPResponse: """On a 304 we will get a new set of headers that we want to update our cached value with, assuming we have one. @@ -480,11 +478,11 @@ class CacheController(object): excluded_headers = ["content-length"] cached_response.headers.update( - dict( - (k, v) + { + k: v for k, v in response.headers.items() # type: ignore[no-untyped-call] if k.lower() not in excluded_headers - ) + } ) # we want a 200 b/c we have content via the cache diff --git a/src/pip/_vendor/cachecontrol/filewrapper.py b/src/pip/_vendor/cachecontrol/filewrapper.py index 472ba6001..25143902a 100644 --- a/src/pip/_vendor/cachecontrol/filewrapper.py +++ b/src/pip/_vendor/cachecontrol/filewrapper.py @@ -1,16 +1,17 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import mmap from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Any, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable if TYPE_CHECKING: from http.client import HTTPResponse -class CallbackFileWrapper(object): +class CallbackFileWrapper: """ Small wrapper around a fp object which will tee everything read into a buffer, and when that file is closed it will execute a callback with the @@ -30,7 +31,7 @@ class CallbackFileWrapper(object): """ def __init__( - self, fp: "HTTPResponse", callback: Optional[Callable[[bytes], None]] + self, fp: HTTPResponse, callback: Callable[[bytes], None] | None ) -> None: self.__buf = NamedTemporaryFile("rb+", delete=True) self.__fp = fp @@ -93,7 +94,7 @@ class CallbackFileWrapper(object): # Important when caching big files. self.__buf.close() - def read(self, amt: Optional[int] = None) -> bytes: + def read(self, amt: int | None = None) -> bytes: data: bytes = self.__fp.read(amt) if data: # We may be dealing with b'', a sign that things are over: diff --git a/src/pip/_vendor/cachecontrol/heuristics.py b/src/pip/_vendor/cachecontrol/heuristics.py index 1e88ada68..b9d72ca4a 100644 --- a/src/pip/_vendor/cachecontrol/heuristics.py +++ b/src/pip/_vendor/cachecontrol/heuristics.py @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import calendar import time from datetime import datetime, timedelta, timezone from email.utils import formatdate, parsedate, parsedate_tz -from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional +from typing import TYPE_CHECKING, Any, Mapping if TYPE_CHECKING: from pip._vendor.urllib3 import HTTPResponse @@ -14,7 +15,7 @@ if TYPE_CHECKING: TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT" -def expire_after(delta: timedelta, date: Optional[datetime] = None) -> datetime: +def expire_after(delta: timedelta, date: datetime | None = None) -> datetime: date = date or datetime.now(timezone.utc) return date + delta @@ -23,8 +24,8 @@ def datetime_to_header(dt: datetime) -> str: return formatdate(calendar.timegm(dt.timetuple())) -class BaseHeuristic(object): - def warning(self, response: "HTTPResponse") -> Optional[str]: +class BaseHeuristic: + def warning(self, response: HTTPResponse) -> str | None: """ Return a valid 1xx warning header value describing the cache adjustments. @@ -35,7 +36,7 @@ class BaseHeuristic(object): """ return '110 - "Response is Stale"' - def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: + def update_headers(self, response: HTTPResponse) -> dict[str, str]: """Update the response headers with any new headers. NOTE: This SHOULD always include some Warning header to @@ -44,7 +45,7 @@ class BaseHeuristic(object): """ return {} - def apply(self, response: "HTTPResponse") -> "HTTPResponse": + def apply(self, response: HTTPResponse) -> HTTPResponse: updated_headers = self.update_headers(response) if updated_headers: @@ -62,7 +63,7 @@ class OneDayCache(BaseHeuristic): future. """ - def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: + def update_headers(self, response: HTTPResponse) -> dict[str, str]: headers = {} if "expires" not in response.headers: @@ -81,11 +82,11 @@ class ExpiresAfter(BaseHeuristic): def __init__(self, **kw: Any) -> None: self.delta = timedelta(**kw) - def update_headers(self, response: "HTTPResponse") -> Dict[str, str]: + def update_headers(self, response: HTTPResponse) -> dict[str, str]: expires = expire_after(self.delta) return {"expires": datetime_to_header(expires), "cache-control": "public"} - def warning(self, response: "HTTPResponse") -> Optional[str]: + def warning(self, response: HTTPResponse) -> str | None: tmpl = "110 - Automatically cached for %s. Response might be stale" return tmpl % self.delta @@ -117,7 +118,7 @@ class LastModified(BaseHeuristic): 501, } - def update_headers(self, resp: "HTTPResponse") -> Dict[str, str]: + def update_headers(self, resp: HTTPResponse) -> dict[str, str]: headers: Mapping[str, str] = resp.headers if "expires" in headers: @@ -149,5 +150,5 @@ class LastModified(BaseHeuristic): expires = date + freshness_lifetime return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))} - def warning(self, resp: "HTTPResponse") -> Optional[str]: + def warning(self, resp: HTTPResponse) -> str | None: return None diff --git a/src/pip/_vendor/cachecontrol/serialize.py b/src/pip/_vendor/cachecontrol/serialize.py index f21eaea6f..f9e967c3c 100644 --- a/src/pip/_vendor/cachecontrol/serialize.py +++ b/src/pip/_vendor/cachecontrol/serialize.py @@ -1,39 +1,27 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -import base64 import io -import json -import pickle -import zlib -from typing import IO, TYPE_CHECKING, Any, Mapping, Optional +from typing import IO, TYPE_CHECKING, Any, Mapping, cast from pip._vendor import msgpack from pip._vendor.requests.structures import CaseInsensitiveDict from pip._vendor.urllib3 import HTTPResponse if TYPE_CHECKING: - from pip._vendor.requests import PreparedRequest, Request + from pip._vendor.requests import PreparedRequest -def _b64_decode_bytes(b: str) -> bytes: - return base64.b64decode(b.encode("ascii")) +class Serializer: + serde_version = "4" - -def _b64_decode_str(s: str) -> str: - return _b64_decode_bytes(s).decode("utf8") - - -_default_body_read = object() - - -class Serializer(object): def dumps( self, - request: "PreparedRequest", + request: PreparedRequest, response: HTTPResponse, - body: Optional[bytes] = None, + body: bytes | None = None, ) -> bytes: response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( response.headers @@ -50,7 +38,7 @@ class Serializer(object): data = { "response": { "body": body, # Empty bytestring if body is stored separately - "headers": dict((str(k), str(v)) for k, v in response.headers.items()), # type: ignore[no-untyped-call] + "headers": {str(k): str(v) for k, v in response.headers.items()}, # type: ignore[no-untyped-call] "status": response.status, "version": response.version, "reason": str(response.reason), @@ -69,14 +57,17 @@ class Serializer(object): header_value = str(header_value) data["vary"][header] = header_value - return b",".join([b"cc=4", msgpack.dumps(data, use_bin_type=True)]) + return b",".join([f"cc={self.serde_version}".encode(), self.serialize(data)]) + + def serialize(self, data: dict[str, Any]) -> bytes: + return cast(bytes, msgpack.dumps(data, use_bin_type=True)) def loads( self, - request: "PreparedRequest", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: # Short circuit if we've been given an empty set of data if not data: return None @@ -99,7 +90,7 @@ class Serializer(object): # Dispatch to the actual load method for the given version try: - return getattr(self, "_loads_v{}".format(verstr))(request, data, body_file) # type: ignore[no-any-return] + return getattr(self, f"_loads_v{verstr}")(request, data, body_file) # type: ignore[no-any-return] except AttributeError: # This is a version we don't have a loads function for, so we'll @@ -108,10 +99,10 @@ class Serializer(object): def prepare_response( self, - request: "Request", + request: PreparedRequest, cached: Mapping[str, Any], - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: """Verify our vary headers match and construct a real urllib3 HTTPResponse object. """ @@ -139,7 +130,7 @@ class Serializer(object): cached["response"]["headers"] = headers try: - body: "IO[bytes]" + body: IO[bytes] if body_file is None: body = io.BytesIO(body_raw) else: @@ -160,71 +151,53 @@ class Serializer(object): def _loads_v0( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, + body_file: IO[bytes] | None = None, ) -> None: # The original legacy cache data. This doesn't contain enough # information to construct everything we need, so we'll treat this as # a miss. - return + return None def _loads_v1( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: - try: - cached = pickle.loads(data) - except ValueError: - return None - - return self.prepare_response(request, cached, body_file) + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: + # The "v1" pickled cache format. This is no longer supported + # for security reasons, so we treat it as a miss. + return None def _loads_v2( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: - assert body_file is None - try: - cached = json.loads(zlib.decompress(data).decode("utf8")) - except (ValueError, zlib.error): - return None - - # We need to decode the items that we've base64 encoded - cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"]) - cached["response"]["headers"] = dict( - (_b64_decode_str(k), _b64_decode_str(v)) - for k, v in cached["response"]["headers"].items() - ) - cached["response"]["reason"] = _b64_decode_str(cached["response"]["reason"]) - cached["vary"] = dict( - (_b64_decode_str(k), _b64_decode_str(v) if v is not None else v) - for k, v in cached["vary"].items() - ) - - return self.prepare_response(request, cached, body_file) + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: + # The "v2" compressed base64 cache format. + # This has been removed due to age and poor size/performance + # characteristics, so we treat it as a miss. + return None def _loads_v3( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, + body_file: IO[bytes] | None = None, ) -> None: # Due to Python 2 encoding issues, it's impossible to know for sure # exactly how to load v3 entries, thus we'll treat these as a miss so # that they get rewritten out as v4 entries. - return + return None def _loads_v4( self, - request: "Request", + request: PreparedRequest, data: bytes, - body_file: Optional["IO[bytes]"] = None, - ) -> Optional[HTTPResponse]: + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: try: cached = msgpack.loads(data, raw=False) except ValueError: diff --git a/src/pip/_vendor/cachecontrol/wrapper.py b/src/pip/_vendor/cachecontrol/wrapper.py index 293e69fe7..f618bc363 100644 --- a/src/pip/_vendor/cachecontrol/wrapper.py +++ b/src/pip/_vendor/cachecontrol/wrapper.py @@ -1,8 +1,9 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -from typing import TYPE_CHECKING, Collection, Optional, Type +from typing import TYPE_CHECKING, Collection from pip._vendor.cachecontrol.adapter import CacheControlAdapter from pip._vendor.cachecontrol.cache import DictCache @@ -17,15 +18,15 @@ if TYPE_CHECKING: def CacheControl( - sess: "requests.Session", - cache: Optional["BaseCache"] = None, + sess: requests.Session, + cache: BaseCache | None = None, cache_etags: bool = True, - serializer: Optional["Serializer"] = None, - heuristic: Optional["BaseHeuristic"] = None, - controller_class: Optional[Type["CacheController"]] = None, - adapter_class: Optional[Type[CacheControlAdapter]] = None, - cacheable_methods: Optional[Collection[str]] = None, -) -> "requests.Session": + serializer: Serializer | None = None, + heuristic: BaseHeuristic | None = None, + controller_class: type[CacheController] | None = None, + adapter_class: type[CacheControlAdapter] | None = None, + cacheable_methods: Collection[str] | None = None, +) -> requests.Session: cache = DictCache() if cache is None else cache adapter_class = adapter_class or CacheControlAdapter adapter = adapter_class( diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index d0f4c71cc..c6809dfd6 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -1,4 +1,4 @@ -CacheControl==0.13.0 # Make sure to update the license in pyproject.toml for this. +CacheControl==0.13.1 # Make sure to update the license in pyproject.toml for this. colorama==0.4.6 distlib==0.3.6 distro==1.8.0 From a0976d8832f52c2f14472f7b20b1cf1776a63ac8 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 27 Jun 2023 14:47:09 +0100 Subject: [PATCH 080/216] Fix lint issues --- tests/unit/test_network_auth.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 5dde6da57..e3cb772bb 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -193,7 +193,7 @@ def test_keyring_get_password( expect: Tuple[Optional[str], Optional[str]], ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth( index_urls=["http://example.com/path2", "http://example.com/path3"], keyring_provider="import", @@ -205,7 +205,7 @@ def test_keyring_get_password( def test_keyring_get_password_after_prompt(monkeypatch: pytest.MonkeyPatch) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(keyring_provider="import") def ask_input(prompt: str) -> str: @@ -221,7 +221,7 @@ def test_keyring_get_password_after_prompt_when_none( monkeypatch: pytest.MonkeyPatch, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(keyring_provider="import") def ask_input(prompt: str) -> str: @@ -242,7 +242,7 @@ def test_keyring_get_password_username_in_index( monkeypatch: pytest.MonkeyPatch, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth( index_urls=["http://user@example.com/path2", "http://example.com/path4"], keyring_provider="import", @@ -278,7 +278,7 @@ def test_keyring_set_password( expect_save: bool, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(prompting=True, keyring_provider="import") monkeypatch.setattr(auth, "_get_url_and_credentials", lambda u: (u, None, None)) monkeypatch.setattr(auth, "_prompt_for_password", lambda *a: creds) @@ -354,7 +354,7 @@ class KeyringModuleV2: def test_keyring_get_credential( monkeypatch: pytest.MonkeyPatch, url: str, expect: str ) -> None: - monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) auth = MultiDomainBasicAuth( index_urls=["http://example.com/path1", "http://example.com/path2"], keyring_provider="import", @@ -378,7 +378,7 @@ class KeyringModuleBroken: def test_broken_keyring_disables_keyring(monkeypatch: pytest.MonkeyPatch) -> None: keyring_broken = KeyringModuleBroken() - monkeypatch.setitem(sys.modules, "keyring", keyring_broken) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring_broken) auth = MultiDomainBasicAuth( index_urls=["http://example.com/"], keyring_provider="import" From c7daa07f6a65c73173f623c1be34ed2956628715 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 27 Jun 2023 14:47:39 +0100 Subject: [PATCH 081/216] Reword the check for no hashes --- src/pip/_internal/models/link.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 3cfc3e8c4..4453519ad 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -112,9 +112,9 @@ def supported_hashes(hashes: Optional[Dict[str, str]]) -> Optional[Dict[str, str if hashes is None: return None hashes = {n: v for n, v in hashes.items() if n in _SUPPORTED_HASHES} - if len(hashes) > 0: - return hashes - return None + if not hashes: + return None + return hashes def _clean_url_path_part(part: str) -> str: From fab8cf7479f573a9284ae4c9a85f776c951c6656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 27 Jun 2023 15:49:40 +0200 Subject: [PATCH 082/216] Remove Unused "type: ignore" comments --- tests/unit/test_network_auth.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 5dde6da57..e3cb772bb 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -193,7 +193,7 @@ def test_keyring_get_password( expect: Tuple[Optional[str], Optional[str]], ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth( index_urls=["http://example.com/path2", "http://example.com/path3"], keyring_provider="import", @@ -205,7 +205,7 @@ def test_keyring_get_password( def test_keyring_get_password_after_prompt(monkeypatch: pytest.MonkeyPatch) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(keyring_provider="import") def ask_input(prompt: str) -> str: @@ -221,7 +221,7 @@ def test_keyring_get_password_after_prompt_when_none( monkeypatch: pytest.MonkeyPatch, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(keyring_provider="import") def ask_input(prompt: str) -> str: @@ -242,7 +242,7 @@ def test_keyring_get_password_username_in_index( monkeypatch: pytest.MonkeyPatch, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth( index_urls=["http://user@example.com/path2", "http://example.com/path4"], keyring_provider="import", @@ -278,7 +278,7 @@ def test_keyring_set_password( expect_save: bool, ) -> None: keyring = KeyringModuleV1() - monkeypatch.setitem(sys.modules, "keyring", keyring) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring) auth = MultiDomainBasicAuth(prompting=True, keyring_provider="import") monkeypatch.setattr(auth, "_get_url_and_credentials", lambda u: (u, None, None)) monkeypatch.setattr(auth, "_prompt_for_password", lambda *a: creds) @@ -354,7 +354,7 @@ class KeyringModuleV2: def test_keyring_get_credential( monkeypatch: pytest.MonkeyPatch, url: str, expect: str ) -> None: - monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", KeyringModuleV2()) auth = MultiDomainBasicAuth( index_urls=["http://example.com/path1", "http://example.com/path2"], keyring_provider="import", @@ -378,7 +378,7 @@ class KeyringModuleBroken: def test_broken_keyring_disables_keyring(monkeypatch: pytest.MonkeyPatch) -> None: keyring_broken = KeyringModuleBroken() - monkeypatch.setitem(sys.modules, "keyring", keyring_broken) # type: ignore[misc] + monkeypatch.setitem(sys.modules, "keyring", keyring_broken) auth = MultiDomainBasicAuth( index_urls=["http://example.com/"], keyring_provider="import" From 782cff7e0121d5160acddbfae2ef41e98492ffe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 10 Apr 2023 15:59:59 +0200 Subject: [PATCH 083/216] Warn when legacy versions and specifiers are resolved Also warn in pip check. ... --- news/12063.removal.rst | 2 ++ src/pip/_internal/commands/check.py | 2 ++ src/pip/_internal/commands/download.py | 1 + src/pip/_internal/commands/install.py | 3 ++ src/pip/_internal/commands/wheel.py | 1 + src/pip/_internal/operations/check.py | 38 ++++++++++++++++++++++++++ src/pip/_internal/req/req_set.py | 37 +++++++++++++++++++++++++ 7 files changed, 84 insertions(+) create mode 100644 news/12063.removal.rst diff --git a/news/12063.removal.rst b/news/12063.removal.rst new file mode 100644 index 000000000..037b0c608 --- /dev/null +++ b/news/12063.removal.rst @@ -0,0 +1,2 @@ +Deprecate legacy version and version specifiers that don't conform to `PEP 440 +`_ diff --git a/src/pip/_internal/commands/check.py b/src/pip/_internal/commands/check.py index 584df9f55..5efd0a341 100644 --- a/src/pip/_internal/commands/check.py +++ b/src/pip/_internal/commands/check.py @@ -7,6 +7,7 @@ from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.operations.check import ( check_package_set, create_package_set_from_installed, + warn_legacy_versions_and_specifiers, ) from pip._internal.utils.misc import write_output @@ -21,6 +22,7 @@ class CheckCommand(Command): def run(self, options: Values, args: List[str]) -> int: package_set, parsing_probs = create_package_set_from_installed() + warn_legacy_versions_and_specifiers(package_set) missing, conflicting = check_package_set(package_set) for project_name in missing: diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 36e947c8c..63bd53a50 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -130,6 +130,7 @@ class DownloadCommand(RequirementCommand): self.trace_basic_info(finder) requirement_set = resolver.resolve(reqs, check_supported_wheels=True) + requirement_set.warn_legacy_versions_and_specifiers() downloaded: List[str] = [] for req in requirement_set.requirements.values(): diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 3c15ed415..f6a300804 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -387,6 +387,9 @@ class InstallCommand(RequirementCommand): json.dump(report.to_dict(), f, indent=2, ensure_ascii=False) if options.dry_run: + # In non dry-run mode, the legacy versions and specifiers check + # will be done as part of conflict detection. + requirement_set.warn_legacy_versions_and_specifiers() would_install_items = sorted( (r.metadata["name"], r.metadata["version"]) for r in requirement_set.requirements_to_install diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index c6a588ff0..e6735bd8d 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -145,6 +145,7 @@ class WheelCommand(RequirementCommand): self.trace_basic_info(finder) requirement_set = resolver.resolve(reqs, check_supported_wheels=True) + requirement_set.warn_legacy_versions_and_specifiers() reqs_to_build: List[InstallRequirement] = [] for req in requirement_set.requirements.values(): diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index e3bce69b2..261045922 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -5,12 +5,15 @@ import logging from typing import Callable, Dict, List, NamedTuple, Optional, Set, Tuple from pip._vendor.packaging.requirements import Requirement +from pip._vendor.packaging.specifiers import LegacySpecifier from pip._vendor.packaging.utils import NormalizedName, canonicalize_name +from pip._vendor.packaging.version import LegacyVersion from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.metadata import get_default_environment from pip._internal.metadata.base import DistributionVersion from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.deprecation import deprecated logger = logging.getLogger(__name__) @@ -57,6 +60,8 @@ def check_package_set( package name and returns a boolean. """ + warn_legacy_versions_and_specifiers(package_set) + missing = {} conflicting = {} @@ -147,3 +152,36 @@ def _create_whitelist( break return packages_affected + + +def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None: + for project_name, package_details in package_set.items(): + if isinstance(package_details.version, LegacyVersion): + deprecated( + reason=( + f"{project_name} {package_details.version} " + f"has a non-standard version number." + ), + replacement=( + f"to upgrade to a newer version of {project_name} " + f"or contact the author to suggest that they " + f"release a version with a conforming version number" + ), + issue=12063, + gone_in="23.3", + ) + for dep in package_details.dependencies: + if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier): + deprecated( + reason=( + f"{project_name} {package_details.version} " + f"has a non-standard dependency specifier {dep}." + ), + replacement=( + f"to upgrade to a newer version of {project_name} " + f"or contact the author to suggest that they " + f"release a version with a conforming dependency specifiers" + ), + issue=12063, + gone_in="23.3", + ) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index ec7a6e07a..cff676017 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -2,9 +2,12 @@ import logging from collections import OrderedDict from typing import Dict, List +from pip._vendor.packaging.specifiers import LegacySpecifier from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.packaging.version import LegacyVersion from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.deprecation import deprecated logger = logging.getLogger(__name__) @@ -80,3 +83,37 @@ class RequirementSet: for install_req in self.all_requirements if not install_req.constraint and not install_req.satisfied_by ] + + def warn_legacy_versions_and_specifiers(self) -> None: + for req in self.requirements_to_install: + version = req.get_dist().version + if isinstance(version, LegacyVersion): + deprecated( + reason=( + f"pip has selected the non standard version {version} " + f"of {req}. In the future this version will be " + f"ignored as it isn't standard compliant." + ), + replacement=( + "set or update constraints to select another version " + "or contact the package author to fix the version number" + ), + issue=12063, + gone_in="23.3", + ) + for dep in req.get_dist().iter_dependencies(): + if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier): + deprecated( + reason=( + f"pip has selected {req} {version} which has non " + f"standard dependency specifier {dep}. " + f"In the future this version of {req} will be " + f"ignored as it isn't standard compliant." + ), + replacement=( + "set or update constraints to select another version " + "or contact the package author to fix the version number" + ), + issue=12063, + gone_in="23.3", + ) From 6507734aac0f3c0972aef5097b8d0b4defb791f8 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 29 Jun 2023 16:51:11 +0800 Subject: [PATCH 084/216] Fix string formatting --- src/pip/_internal/configuration.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 46562652f..35189d0d2 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -216,10 +216,8 @@ class Configuration: parser.write(f) except IOError as error: raise ConfigurationError( - "An error occurred while writing to the configuration file: {0}\n \ - Error message: {1}".format( - fname, error - ) + f"An error occurred while writing to the configuration file " + f"{fname}: {error}" ) # From 41f138e43a6c54804b7c7fe3d4a8477508ef4a97 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 29 Jun 2023 16:51:34 +0800 Subject: [PATCH 085/216] Minimize changeset --- src/pip/_internal/configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 35189d0d2..0ed33ac9c 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -347,7 +347,6 @@ class Configuration: should_load_user_config = not self.isolated and not ( config_file and os.path.exists(config_file) ) - if should_load_user_config: # The legacy config file is overridden by the new config file yield kinds.USER, config_files[kinds.USER] From 256af8f6912799ebf66b2ebc6707aae2e0487fe1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 29 Jun 2023 17:28:16 +0800 Subject: [PATCH 086/216] Catch OSError instead of IOError --- src/pip/_internal/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index 0ed33ac9c..96f824955 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -214,7 +214,7 @@ class Configuration: try: with open(fname, "w") as f: parser.write(f) - except IOError as error: + except OSError as error: raise ConfigurationError( f"An error occurred while writing to the configuration file " f"{fname}: {error}" From 5d0c2773b8bf0cc569c82e6d3697fe7b89c71192 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 30 Jun 2023 11:38:12 +0100 Subject: [PATCH 087/216] Stop using a RAM disk for the Windows tests --- .github/workflows/ci.yml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b35e93b2..136198056 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,24 +167,13 @@ jobs: with: python-version: ${{ matrix.python }} - # We use a RAMDisk on Windows, since filesystem IO is a big slowdown - # for our tests. - - name: Create a RAMDisk - run: ./tools/ci/New-RAMDisk.ps1 -Drive R -Size 1GB - - - name: Setup RAMDisk permissions - run: | - mkdir R:\Temp - $acl = Get-Acl "R:\Temp" - $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( - "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow" - ) - $acl.AddAccessRule($rule) - Set-Acl "R:\Temp" $acl - + # We use C:\Temp (which is already available on the worker) + # as a temporary directory for all of the tests because the + # default value (under the user dir) is more deeply nested + # and causes tests to fail with "path too long" errors. - run: pip install nox env: - TEMP: "R:\\Temp" + TEMP: "C:\\Temp" # Main check - name: Run unit tests @@ -194,7 +183,7 @@ jobs: -m unit --verbose --numprocesses auto --showlocals env: - TEMP: "R:\\Temp" + TEMP: "C:\\Temp" - name: Run integration tests (group 1) if: matrix.group == 1 @@ -203,7 +192,7 @@ jobs: -m integration -k "not test_install" --verbose --numprocesses auto --showlocals env: - TEMP: "R:\\Temp" + TEMP: "C:\\Temp" - name: Run integration tests (group 2) if: matrix.group == 2 @@ -212,7 +201,7 @@ jobs: -m integration -k "test_install" --verbose --numprocesses auto --showlocals env: - TEMP: "R:\\Temp" + TEMP: "C:\\Temp" tests-zipapp: name: tests / zipapp From 45468f06d429080a9042909b76cfc25fce9bee5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 29 May 2023 13:57:52 +0200 Subject: [PATCH 088/216] Pass revisions options explicitly to mercurial commands --- news/12119.bugfix.rst | 3 +++ src/pip/_internal/vcs/mercurial.py | 2 +- tests/unit/test_vcs.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 news/12119.bugfix.rst diff --git a/news/12119.bugfix.rst b/news/12119.bugfix.rst new file mode 100644 index 000000000..da8d8b04d --- /dev/null +++ b/news/12119.bugfix.rst @@ -0,0 +1,3 @@ +Pass the ``-r`` flag to mercurial to be explicit that a revision is passed and protect +against ``hg`` options injection as part of VCS URLs. Users that do not have control on +VCS URLs passed to pip are advised to upgrade. diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 2a005e0af..4595960b5 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -31,7 +31,7 @@ class Mercurial(VersionControl): @staticmethod def get_base_rev_args(rev: str) -> List[str]: - return [rev] + return ["-r", rev] def fetch_new( self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 566c88cf0..38daaa0f2 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -66,7 +66,7 @@ def test_rev_options_repr() -> None: # First check VCS-specific RevOptions behavior. (Bazaar, [], ["-r", "123"], {}), (Git, ["HEAD"], ["123"], {}), - (Mercurial, [], ["123"], {}), + (Mercurial, [], ["-r", "123"], {}), (Subversion, [], ["-r", "123"], {}), # Test extra_args. For this, test using a single VersionControl class. ( From b99e082b003788f2e8abbad47d461f495faad892 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sat, 1 Jul 2023 14:42:10 +0100 Subject: [PATCH 089/216] Record download of completed partial requirements --- src/pip/_internal/operations/prepare.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 130b97377..8d7151353 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -471,6 +471,7 @@ class RequirementPreparer: logger.debug("Downloading link %s to %s", link, filepath) req = links_to_fully_download[link] req.local_file_path = filepath + self._downloaded[req.link.url] = filepath # This step is necessary to ensure all lazy wheels are processed # successfully by the 'download', 'wheel', and 'install' commands. From cb25bf3731d46697586fc72a24ba1f8e57311377 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sat, 1 Jul 2023 14:51:10 +0100 Subject: [PATCH 090/216] Add a news file --- news/11847.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11847.bugfix.rst diff --git a/news/11847.bugfix.rst b/news/11847.bugfix.rst new file mode 100644 index 000000000..1cad477ea --- /dev/null +++ b/news/11847.bugfix.rst @@ -0,0 +1 @@ +Prevent downloading files twice when PEP 658 metadata is present From 647ba8d07e7832ea69d93f9a686d8f276e669a14 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 3 Jul 2023 10:35:01 +0100 Subject: [PATCH 091/216] Limit the double download fix to wheels --- src/pip/_internal/operations/prepare.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 8d7151353..5d9bedc03 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -471,7 +471,19 @@ class RequirementPreparer: logger.debug("Downloading link %s to %s", link, filepath) req = links_to_fully_download[link] req.local_file_path = filepath - self._downloaded[req.link.url] = filepath + # TODO: This needs fixing for sdists + # This is an emergency fix for #11847, which reports that + # distributions get downloaded twice when metadata is loaded + # from a PEP 658 standalone metadata file. Setting _downloaded + # fixes this for wheels, but breaks the sdist case (tests + # test_download_metadata). As PyPI is currently not serving + # metadata for wheels, this is not an immediate issue. + # Fixing the problem properly looks like it will require a + # complete refactoring of the `prepare_linked_requirements_more` + # logic, and I haven't a clue where to start on that, so for now + # I have fixed the issue *just* for wheels. + if req.is_wheel: + self._downloaded[req.link.url] = filepath # This step is necessary to ensure all lazy wheels are processed # successfully by the 'download', 'wheel', and 'install' commands. From 8e80a3ad9a5b80de72efad6cbad3bebf2328642b Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 3 Jul 2023 10:45:01 +0100 Subject: [PATCH 092/216] Fix typo --- src/pip/_internal/operations/prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 5d9bedc03..49d86268a 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -476,7 +476,7 @@ class RequirementPreparer: # distributions get downloaded twice when metadata is loaded # from a PEP 658 standalone metadata file. Setting _downloaded # fixes this for wheels, but breaks the sdist case (tests - # test_download_metadata). As PyPI is currently not serving + # test_download_metadata). As PyPI is currently only serving # metadata for wheels, this is not an immediate issue. # Fixing the problem properly looks like it will require a # complete refactoring of the `prepare_linked_requirements_more` From 82c6a8c9b69abd8f085836f5c362942c9b27bb00 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 Jul 2023 15:35:47 +0800 Subject: [PATCH 093/216] Add 3.12 to noxfile --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index ee03447d3..e64686dca 100644 --- a/noxfile.py +++ b/noxfile.py @@ -67,7 +67,7 @@ def should_update_common_wheels() -> bool: # ----------------------------------------------------------------------------- # Development Commands # ----------------------------------------------------------------------------- -@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3"]) +@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3"]) def test(session: nox.Session) -> None: # Get the common wheels. if should_update_common_wheels(): From c05884a7144012c5bea1ef9c64d568d5649f6cd1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 Jul 2023 15:49:48 +0800 Subject: [PATCH 094/216] Ensure setuptools and wheel are installed when needed --- noxfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/noxfile.py b/noxfile.py index e64686dca..041d90399 100644 --- a/noxfile.py +++ b/noxfile.py @@ -89,6 +89,7 @@ def test(session: nox.Session) -> None: shutil.rmtree(sdist_dir, ignore_errors=True) # fmt: off + session.install("setuptools") session.run( "python", "setup.py", "sdist", "--formats=zip", "--dist-dir", sdist_dir, silent=True, @@ -351,6 +352,7 @@ def build_dists(session: nox.Session) -> List[str]: ) session.log("# Build distributions") + session.install("setuptools", "wheel") session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True) produced_dists = glob.glob("dist/*") From 4abb8ac07b4f532a592555bb6b7196c066f572c6 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 Jul 2023 15:53:01 +0800 Subject: [PATCH 095/216] Fix Pytest --use-venv init --- tests/lib/venv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/venv.py b/tests/lib/venv.py index e65a32912..12aa739db 100644 --- a/tests/lib/venv.py +++ b/tests/lib/venv.py @@ -124,7 +124,7 @@ class VirtualEnvironment: ) elif self._venv_type == "venv": builder = _venv.EnvBuilder() - context = builder.ensure_directories(self.location) + context = builder.ensure_directories(os.fspath(self.location)) builder.create_configuration(context) builder.setup_python(context) self.site.mkdir(parents=True, exist_ok=True) From 56f8b38e1e791bbfc18ec9aae68c113c84d3187b Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 Jul 2023 17:01:08 +0800 Subject: [PATCH 096/216] Fix type handling --- tests/conftest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 57dd7e68a..a183cadf2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,7 +49,7 @@ from .lib.compat import nullcontext if TYPE_CHECKING: from typing import Protocol - from wsgi import WSGIApplication + from _typeshed.wsgi import WSGIApplication else: # TODO: Protocol was introduced in Python 3.8. Remove this branch when # dropping support for Python 3.7. @@ -645,7 +645,12 @@ class InMemoryPip: try: returncode = pip_entry_point([os.fspath(a) for a in args]) except SystemExit as e: - returncode = e.code or 0 + if isinstance(e.code, int): + returncode = e.code + elif e.code: + returncode = 1 + else: + returncode = 0 finally: sys.stdout = orig_stdout return InMemoryPipResult(returncode, stdout.getvalue()) From 4a014f953da027cb314a995ee16f0e28fb821915 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 Jul 2023 17:03:11 +0800 Subject: [PATCH 097/216] Replace distutils with sysconfig on Python 3.12+ --- tests/functional/test_build_env.py | 41 +++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tests/functional/test_build_env.py b/tests/functional/test_build_env.py index 22a71cd32..20cd181be 100644 --- a/tests/functional/test_build_env.py +++ b/tests/functional/test_build_env.py @@ -1,4 +1,5 @@ import os +import sys from textwrap import dedent from typing import Optional @@ -203,6 +204,31 @@ def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> No assert result.stdout.strip() == "2.0", str(result) +if sys.version_info < (3, 12): + BUILD_ENV_ERROR_DEBUG_CODE = r""" + from distutils.sysconfig import get_python_lib + print( + f'imported `pkg` from `{pkg.__file__}`', + file=sys.stderr) + print('system sites:\n ' + '\n '.join(sorted({ + get_python_lib(plat_specific=0), + get_python_lib(plat_specific=1), + })), file=sys.stderr) + """ +else: + BUILD_ENV_ERROR_DEBUG_CODE = r""" + from sysconfig import get_paths + paths = get_paths() + print( + f'imported `pkg` from `{pkg.__file__}`', + file=sys.stderr) + print('system sites:\n ' + '\n '.join(sorted({ + paths['platlib'], + paths['purelib'], + })), file=sys.stderr) + """ + + @pytest.mark.usefixtures("enable_user_site") def test_build_env_isolation(script: PipTestEnvironment) -> None: # Create dummy `pkg` wheel. @@ -231,8 +257,7 @@ def test_build_env_isolation(script: PipTestEnvironment) -> None: run_with_build_env( script, "", - r""" - from distutils.sysconfig import get_python_lib + f""" import sys try: @@ -240,17 +265,9 @@ def test_build_env_isolation(script: PipTestEnvironment) -> None: except ImportError: pass else: - print( - f'imported `pkg` from `{pkg.__file__}`', - file=sys.stderr) - print('system sites:\n ' + '\n '.join(sorted({ - get_python_lib(plat_specific=0), - get_python_lib(plat_specific=1), - })), file=sys.stderr) - print('sys.path:\n ' + '\n '.join(sys.path), file=sys.stderr) + {BUILD_ENV_ERROR_DEBUG_CODE} + print('sys.path:\\n ' + '\\n '.join(sys.path), file=sys.stderr) sys.exit(1) - """ - f""" # second check: direct check of exclusion of system site packages import os From 60d3c0447ef6ea659f6f4666758e50c0ba99fb69 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 Jul 2023 17:04:09 +0800 Subject: [PATCH 098/216] Skip tests that do not make sense on Python 3.12+ --- tests/functional/test_install.py | 20 +++++++++++++++----- tests/functional/test_uninstall.py | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 8559d9368..eabddfe58 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -848,14 +848,18 @@ def test_editable_install__local_dir_no_setup_py( ) +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Setuptools<64 does not support Python 3.12+", +) @pytest.mark.network -def test_editable_install__local_dir_no_setup_py_with_pyproject( +def test_editable_install_legacy__local_dir_no_setup_py_with_pyproject( script: PipTestEnvironment, ) -> None: """ - Test installing in editable mode from a local directory with no setup.py - but that does have pyproject.toml with a build backend that does not support - the build_editable hook. + Test installing in legacy editable mode from a local directory with no + setup.py but that does have pyproject.toml with a build backend that does + not support the build_editable hook. """ local_dir = script.scratch_path.joinpath("temp") local_dir.mkdir() @@ -1383,8 +1387,14 @@ setup(name='pkga', version='0.1') _test_install_editable_with_prefix(script, {"setup.py": setup_py}) +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Setuptools<64 does not support Python 3.12+", +) @pytest.mark.network -def test_install_editable_with_prefix_setup_cfg(script: PipTestEnvironment) -> None: +def test_install_editable_legacy_with_prefix_setup_cfg( + script: PipTestEnvironment, +) -> None: setup_cfg = """[metadata] name = pkga version = 0.1 diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index 87e715749..be7fe4c33 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -37,6 +37,10 @@ def test_basic_uninstall(script: PipTestEnvironment) -> None: assert_all_changes(result, result2, [script.venv / "build", "cache"]) +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="distutils is no longer available in Python 3.12+", +) def test_basic_uninstall_distutils(script: PipTestEnvironment) -> None: """ Test basic install and uninstall. @@ -68,6 +72,10 @@ def test_basic_uninstall_distutils(script: PipTestEnvironment) -> None: ) in result.stderr +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Setuptools<64 does not support Python 3.12+", +) @pytest.mark.network def test_basic_uninstall_with_scripts(script: PipTestEnvironment) -> None: """ @@ -101,6 +109,10 @@ def test_uninstall_invalid_parameter( assert expected_message in result.stderr +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Setuptools<64 does not support Python 3.12+", +) @pytest.mark.network def test_uninstall_easy_install_after_import(script: PipTestEnvironment) -> None: """ @@ -126,6 +138,10 @@ def test_uninstall_easy_install_after_import(script: PipTestEnvironment) -> None ) +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Setuptools<64 does not support Python 3.12+", +) @pytest.mark.network def test_uninstall_trailing_newline(script: PipTestEnvironment) -> None: """ @@ -337,6 +353,10 @@ def test_uninstall_console_scripts_uppercase_name(script: PipTestEnvironment) -> assert not script_name.exists() +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Setuptools<64 does not support Python 3.12+", +) @pytest.mark.network def test_uninstall_easy_installed_console_scripts(script: PipTestEnvironment) -> None: """ From 37640292a17e40fc980988f7cc01b4bb544fe477 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 Jul 2023 17:13:39 +0800 Subject: [PATCH 099/216] Add CPython 3.12 to documentaiton --- docs/html/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/html/installation.md b/docs/html/installation.md index 036a91397..c61fceaed 100644 --- a/docs/html/installation.md +++ b/docs/html/installation.md @@ -103,7 +103,7 @@ $ pip install --upgrade pip The current version of pip works on: - Windows, Linux and MacOS. -- CPython 3.7, 3.8, 3.9, 3.10 and latest PyPy3. +- CPython 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, and latest PyPy3. pip is tested to work on the latest patch version of the Python interpreter, for each of the minor versions listed above. Previous patch versions are From e29dc1cb0a775b761d71ec659cc49f99bd6c23fb Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 4 Jul 2023 17:06:04 +0800 Subject: [PATCH 100/216] Add Python 3.12 to CI --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 136198056..c9a2ff665 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: - run: git diff --exit-code tests-unix: - name: tests / ${{ matrix.python }} / ${{ matrix.os }} + name: tests / ${{ matrix.python.key || matrix.python }} / ${{ matrix.os }} runs-on: ${{ matrix.os }}-latest needs: [packaging, determine-changes] @@ -109,12 +109,14 @@ jobs: - "3.9" - "3.10" - "3.11" + - key: "3.12" + full: "3.12.0-beta.3" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python }} + python-version: ${{ matrix.python.full || matrix.python }} - name: Install Ubuntu dependencies if: matrix.os == 'Ubuntu' @@ -129,12 +131,12 @@ jobs: # Main check - name: Run unit tests run: >- - nox -s test-${{ matrix.python }} -- + nox -s test-${{ matrix.python.key || matrix.python }} -- -m unit --verbose --numprocesses auto --showlocals - name: Run integration tests run: >- - nox -s test-${{ matrix.python }} -- + nox -s test-${{ matrix.python.key || matrix.python }} -- -m integration --verbose --numprocesses auto --showlocals --durations=5 From 5dc65eabb75f89d4f4749b6c764042c227f6870a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD=20=D0=94=D0=BE=D0=BD=D1=87?= =?UTF-8?q?=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Mon, 15 May 2023 22:42:51 +0400 Subject: [PATCH 101/216] Don't exclude setuptools, distribute & wheel from freeze output on Python 3.12+ Due to the advent of build isolation, it is no longer necessary to install setuptools and wheel in an environment just to install other packages. Moreover, on Python 3.12 both ensurepip [1] and virtualenv [2] are to stop installing setuptools & wheel by default. This means that when those packages are present in a Python 3.12+ environment, it is reasonable to assume that they are runtime dependencies of the user's project, and therefore should be included in freeze output. distribute is just obsolete. [1] https://github.com/python/cpython/issues/95299 [2] https://github.com/pypa/virtualenv/pull/2558 --- news/4256.removal.rst | 3 +++ src/pip/_internal/commands/freeze.py | 5 ++++- tests/functional/test_freeze.py | 20 +++++++++++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 news/4256.removal.rst diff --git a/news/4256.removal.rst b/news/4256.removal.rst new file mode 100644 index 000000000..5440f532a --- /dev/null +++ b/news/4256.removal.rst @@ -0,0 +1,3 @@ +``freeze`` no longer excludes the ``setuptools``, ``distribute`` and ``wheel`` +packages from the output by default when running on Python 3.12 or later. +Use ``--exclude`` if you wish to exclude any of these packages. diff --git a/src/pip/_internal/commands/freeze.py b/src/pip/_internal/commands/freeze.py index 5fa6d39b2..87f281d76 100644 --- a/src/pip/_internal/commands/freeze.py +++ b/src/pip/_internal/commands/freeze.py @@ -8,7 +8,10 @@ from pip._internal.cli.status_codes import SUCCESS from pip._internal.operations.freeze import freeze from pip._internal.utils.compat import stdlib_pkgs -DEV_PKGS = {"pip", "setuptools", "distribute", "wheel"} +DEV_PKGS = {"pip"} + +if sys.version_info < (3, 12): + DEV_PKGS |= {"setuptools", "distribute", "wheel"} class FreezeCommand(Command): diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index b24b27edc..81a660ab6 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -88,11 +88,29 @@ def test_basic_freeze(script: PipTestEnvironment) -> None: def test_freeze_with_pip(script: PipTestEnvironment) -> None: - """Test pip shows itself""" + """Test that pip shows itself only when --all is used""" + result = script.pip("freeze") + assert "pip==" not in result.stdout result = script.pip("freeze", "--all") assert "pip==" in result.stdout +def test_freeze_with_setuptools(script: PipTestEnvironment) -> None: + """ + Test that pip shows setuptools only when --all is used + or Python version is >=3.12 + """ + + result = script.pip("freeze") + if sys.version_info >= (3, 12): + assert "setuptools==" in result.stdout + else: + assert "setuptools==" not in result.stdout + + result = script.pip("freeze", "--all") + assert "setuptools==" in result.stdout + + def test_exclude_and_normalization(script: PipTestEnvironment, tmpdir: Path) -> None: req_path = wheel.make_wheel(name="Normalizable_Name", version="1.0").save_to_dir( tmpdir From 393ccfbc31eccdf7f053ee4d62b055e515ef3183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD=20=D0=94=D0=BE=D0=BD=D1=87?= =?UTF-8?q?=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Mon, 29 May 2023 02:23:01 +0400 Subject: [PATCH 102/216] test_freeze_with_setuptools: use mocks This makes it possible to test both branches on any Python version. --- src/pip/_internal/commands/freeze.py | 20 +++++++++++----- tests/functional/test_freeze.py | 34 ++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/commands/freeze.py b/src/pip/_internal/commands/freeze.py index 87f281d76..fd9d88a8b 100644 --- a/src/pip/_internal/commands/freeze.py +++ b/src/pip/_internal/commands/freeze.py @@ -1,6 +1,6 @@ import sys from optparse import Values -from typing import List +from typing import AbstractSet, List from pip._internal.cli import cmdoptions from pip._internal.cli.base_command import Command @@ -8,10 +8,18 @@ from pip._internal.cli.status_codes import SUCCESS from pip._internal.operations.freeze import freeze from pip._internal.utils.compat import stdlib_pkgs -DEV_PKGS = {"pip"} -if sys.version_info < (3, 12): - DEV_PKGS |= {"setuptools", "distribute", "wheel"} +def _should_suppress_build_backends() -> bool: + return sys.version_info < (3, 12) + + +def _dev_pkgs() -> AbstractSet[str]: + pkgs = {"pip"} + + if _should_suppress_build_backends(): + pkgs |= {"setuptools", "distribute", "wheel"} + + return pkgs class FreezeCommand(Command): @@ -64,7 +72,7 @@ class FreezeCommand(Command): action="store_true", help=( "Do not skip these packages in the output:" - " {}".format(", ".join(DEV_PKGS)) + " {}".format(", ".join(_dev_pkgs())) ), ) self.cmd_opts.add_option( @@ -80,7 +88,7 @@ class FreezeCommand(Command): def run(self, options: Values, args: List[str]) -> int: skip = set(stdlib_pkgs) if not options.freeze_all: - skip.update(DEV_PKGS) + skip.update(_dev_pkgs()) if options.excludes: skip.update(options.excludes) diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 81a660ab6..d6122308a 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -98,18 +98,38 @@ def test_freeze_with_pip(script: PipTestEnvironment) -> None: def test_freeze_with_setuptools(script: PipTestEnvironment) -> None: """ Test that pip shows setuptools only when --all is used - or Python version is >=3.12 + or _should_suppress_build_backends() returns false """ - result = script.pip("freeze") - if sys.version_info >= (3, 12): - assert "setuptools==" in result.stdout - else: - assert "setuptools==" not in result.stdout - result = script.pip("freeze", "--all") assert "setuptools==" in result.stdout + (script.site_packages_path / "mock.pth").write_text("import mock\n") + + (script.site_packages_path / "mock.py").write_text( + textwrap.dedent( + """\ + import pip._internal.commands.freeze as freeze + freeze._should_suppress_build_backends = lambda: False + """ + ) + ) + + result = script.pip("freeze") + assert "setuptools==" in result.stdout + + (script.site_packages_path / "mock.py").write_text( + textwrap.dedent( + """\ + import pip._internal.commands.freeze as freeze + freeze._should_suppress_build_backends = lambda: True + """ + ) + ) + + result = script.pip("freeze") + assert "setuptools==" not in result.stdout + def test_exclude_and_normalization(script: PipTestEnvironment, tmpdir: Path) -> None: req_path = wheel.make_wheel(name="Normalizable_Name", version="1.0").save_to_dir( From 7a69c00720fb8e660ef0d1df174b79e039bdba95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD=20=D0=94=D0=BE=D0=BD=D1=87?= =?UTF-8?q?=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Fri, 7 Jul 2023 01:48:03 +0300 Subject: [PATCH 103/216] Make the changelog entry more verbose Co-authored-by: Tzu-ping Chung --- news/4256.removal.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/news/4256.removal.rst b/news/4256.removal.rst index 5440f532a..eb8989850 100644 --- a/news/4256.removal.rst +++ b/news/4256.removal.rst @@ -1,3 +1,4 @@ -``freeze`` no longer excludes the ``setuptools``, ``distribute`` and ``wheel`` -packages from the output by default when running on Python 3.12 or later. -Use ``--exclude`` if you wish to exclude any of these packages. +``freeze`` no longer excludes the ``setuptools``, ``distribute``, and ``wheel`` +from the output when running on Python 3.12 or later, where they are not +included in a virtual environment by default. Use ``--exclude`` if you wish to +exclude any of these packages. From 856c7ec27e8c1dc58a32191c1bcbbc151727a893 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 9 Jul 2023 21:30:08 +0100 Subject: [PATCH 104/216] Upgrade platformdirs to 3.8.1 --- news/platformdirs.vendor.rst | 1 + src/pip/_vendor/platformdirs/__init__.py | 143 +++++++++++++++-------- src/pip/_vendor/platformdirs/__main__.py | 24 ++-- src/pip/_vendor/platformdirs/android.py | 112 +++++++++++++++--- src/pip/_vendor/platformdirs/api.py | 70 ++++++++--- src/pip/_vendor/platformdirs/macos.py | 33 +++++- src/pip/_vendor/platformdirs/unix.py | 97 +++++++++------ src/pip/_vendor/platformdirs/version.py | 4 +- src/pip/_vendor/platformdirs/windows.py | 106 +++++++++++++---- src/pip/_vendor/vendor.txt | 2 +- 10 files changed, 442 insertions(+), 150 deletions(-) create mode 100644 news/platformdirs.vendor.rst diff --git a/news/platformdirs.vendor.rst b/news/platformdirs.vendor.rst new file mode 100644 index 000000000..f396d84a6 --- /dev/null +++ b/news/platformdirs.vendor.rst @@ -0,0 +1 @@ +Upgrade platformdirs to 3.8.1 diff --git a/src/pip/_vendor/platformdirs/__init__.py b/src/pip/_vendor/platformdirs/__init__.py index c46a145cd..5ebf5957b 100644 --- a/src/pip/_vendor/platformdirs/__init__.py +++ b/src/pip/_vendor/platformdirs/__init__.py @@ -6,17 +6,20 @@ from __future__ import annotations import os import sys -from pathlib import Path - -if sys.version_info >= (3, 8): # pragma: no cover (py38+) - from typing import Literal -else: # pragma: no cover (py38+) - from pip._vendor.typing_extensions import Literal +from typing import TYPE_CHECKING from .api import PlatformDirsABC from .version import __version__ from .version import __version_tuple__ as __version_info__ +if TYPE_CHECKING: + from pathlib import Path + + if sys.version_info >= (3, 8): # pragma: no cover (py38+) + from typing import Literal + else: # pragma: no cover (py38+) + from pip._vendor.typing_extensions import Literal + def _set_platform_dir_class() -> type[PlatformDirsABC]: if sys.platform == "win32": @@ -48,8 +51,8 @@ def user_data_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -72,8 +75,8 @@ def site_data_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - multipath: bool = False, - ensure_exists: bool = False, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -96,8 +99,8 @@ def user_config_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -120,8 +123,8 @@ def site_config_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - multipath: bool = False, - ensure_exists: bool = False, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -144,8 +147,8 @@ def user_cache_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -168,8 +171,8 @@ def site_cache_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -192,8 +195,8 @@ def user_state_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -216,8 +219,8 @@ def user_log_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -237,18 +240,36 @@ def user_log_dir( def user_documents_dir() -> str: - """ - :returns: documents directory tied to the user - """ + """:returns: documents directory tied to the user""" return PlatformDirs().user_documents_dir +def user_downloads_dir() -> str: + """:returns: downloads directory tied to the user""" + return PlatformDirs().user_downloads_dir + + +def user_pictures_dir() -> str: + """:returns: pictures directory tied to the user""" + return PlatformDirs().user_pictures_dir + + +def user_videos_dir() -> str: + """:returns: videos directory tied to the user""" + return PlatformDirs().user_videos_dir + + +def user_music_dir() -> str: + """:returns: music directory tied to the user""" + return PlatformDirs().user_music_dir + + def user_runtime_dir( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> str: """ :param appname: See `appname `. @@ -271,8 +292,8 @@ def user_data_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -295,8 +316,8 @@ def site_data_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - multipath: bool = False, - ensure_exists: bool = False, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -319,8 +340,8 @@ def user_config_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -343,8 +364,8 @@ def site_config_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - multipath: bool = False, - ensure_exists: bool = False, + multipath: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -367,8 +388,8 @@ def site_cache_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -391,8 +412,8 @@ def user_cache_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -415,8 +436,8 @@ def user_state_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - ensure_exists: bool = False, + roaming: bool = False, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -439,8 +460,8 @@ def user_log_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -460,18 +481,36 @@ def user_log_path( def user_documents_path() -> Path: - """ - :returns: documents path tied to the user - """ + """:returns: documents path tied to the user""" return PlatformDirs().user_documents_path +def user_downloads_path() -> Path: + """:returns: downloads path tied to the user""" + return PlatformDirs().user_downloads_path + + +def user_pictures_path() -> Path: + """:returns: pictures path tied to the user""" + return PlatformDirs().user_pictures_path + + +def user_videos_path() -> Path: + """:returns: videos path tied to the user""" + return PlatformDirs().user_videos_path + + +def user_music_path() -> Path: + """:returns: music path tied to the user""" + return PlatformDirs().user_music_path + + def user_runtime_path( appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - opinion: bool = True, - ensure_exists: bool = False, + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 ) -> Path: """ :param appname: See `appname `. @@ -502,6 +541,10 @@ __all__ = [ "user_state_dir", "user_log_dir", "user_documents_dir", + "user_downloads_dir", + "user_pictures_dir", + "user_videos_dir", + "user_music_dir", "user_runtime_dir", "site_data_dir", "site_config_dir", @@ -512,6 +555,10 @@ __all__ = [ "user_state_path", "user_log_path", "user_documents_path", + "user_downloads_path", + "user_pictures_path", + "user_videos_path", + "user_music_path", "user_runtime_path", "site_data_path", "site_config_path", diff --git a/src/pip/_vendor/platformdirs/__main__.py b/src/pip/_vendor/platformdirs/__main__.py index 7171f1311..6a0d6dd12 100644 --- a/src/pip/_vendor/platformdirs/__main__.py +++ b/src/pip/_vendor/platformdirs/__main__.py @@ -1,3 +1,4 @@ +"""Main entry point.""" from __future__ import annotations from pip._vendor.platformdirs import PlatformDirs, __version__ @@ -9,6 +10,10 @@ PROPS = ( "user_state_dir", "user_log_dir", "user_documents_dir", + "user_downloads_dir", + "user_pictures_dir", + "user_videos_dir", + "user_music_dir", "user_runtime_dir", "site_data_dir", "site_config_dir", @@ -17,30 +22,31 @@ PROPS = ( def main() -> None: + """Run main entry point.""" app_name = "MyApp" app_author = "MyCompany" - print(f"-- platformdirs {__version__} --") + print(f"-- platformdirs {__version__} --") # noqa: T201 - print("-- app dirs (with optional 'version')") + print("-- app dirs (with optional 'version')") # noqa: T201 dirs = PlatformDirs(app_name, app_author, version="1.0") for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 - print("\n-- app dirs (without optional 'version')") + print("\n-- app dirs (without optional 'version')") # noqa: T201 dirs = PlatformDirs(app_name, app_author) for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 - print("\n-- app dirs (without optional 'appauthor')") + print("\n-- app dirs (without optional 'appauthor')") # noqa: T201 dirs = PlatformDirs(app_name) for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 - print("\n-- app dirs (with disabled 'appauthor')") + print("\n-- app dirs (with disabled 'appauthor')") # noqa: T201 dirs = PlatformDirs(app_name, appauthor=False) for prop in PROPS: - print(f"{prop}: {getattr(dirs, prop)}") + print(f"{prop}: {getattr(dirs, prop)}") # noqa: T201 if __name__ == "__main__": diff --git a/src/pip/_vendor/platformdirs/android.py b/src/pip/_vendor/platformdirs/android.py index f6de7451b..76527dda4 100644 --- a/src/pip/_vendor/platformdirs/android.py +++ b/src/pip/_vendor/platformdirs/android.py @@ -1,3 +1,4 @@ +"""Android.""" from __future__ import annotations import os @@ -30,7 +31,8 @@ class Android(PlatformDirsABC): @property def user_config_dir(self) -> str: """ - :return: config directory tied to the user, e.g. ``/data/user///shared_prefs/`` + :return: config directory tied to the user, e.g. \ + ``/data/user///shared_prefs/`` """ return self._append_app_name_and_version(cast(str, _android_folder()), "shared_prefs") @@ -62,16 +64,34 @@ class Android(PlatformDirsABC): """ path = self.user_cache_dir if self.opinion: - path = os.path.join(path, "log") + path = os.path.join(path, "log") # noqa: PTH118 return path @property def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user e.g. ``/storage/emulated/0/Documents`` - """ + """:return: documents directory tied to the user e.g. ``/storage/emulated/0/Documents``""" return _android_documents_folder() + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user e.g. ``/storage/emulated/0/Downloads``""" + return _android_downloads_folder() + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user e.g. ``/storage/emulated/0/Pictures``""" + return _android_pictures_folder() + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user e.g. ``/storage/emulated/0/DCIM/Camera``""" + return _android_videos_folder() + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user e.g. ``/storage/emulated/0/Music``""" + return _android_music_folder() + @property def user_runtime_dir(self) -> str: """ @@ -80,20 +100,20 @@ class Android(PlatformDirsABC): """ path = self.user_cache_dir if self.opinion: - path = os.path.join(path, "tmp") + path = os.path.join(path, "tmp") # noqa: PTH118 return path @lru_cache(maxsize=1) def _android_folder() -> str | None: - """:return: base folder for the Android OS or None if cannot be found""" + """:return: base folder for the Android OS or None if it cannot be found""" try: # First try to get path to android app via pyjnius from jnius import autoclass - Context = autoclass("android.content.Context") # noqa: N806 - result: str | None = Context.getFilesDir().getParentFile().getAbsolutePath() - except Exception: + context = autoclass("android.content.Context") + result: str | None = context.getFilesDir().getParentFile().getAbsolutePath() + except Exception: # noqa: BLE001 # if fails find an android folder looking path on the sys.path pattern = re.compile(r"/data/(data|user/\d+)/(.+)/files") for path in sys.path: @@ -112,15 +132,79 @@ def _android_documents_folder() -> str: try: from jnius import autoclass - Context = autoclass("android.content.Context") # noqa: N806 - Environment = autoclass("android.os.Environment") # noqa: N806 - documents_dir: str = Context.getExternalFilesDir(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath() - except Exception: + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + documents_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DOCUMENTS).getAbsolutePath() + except Exception: # noqa: BLE001 documents_dir = "/storage/emulated/0/Documents" return documents_dir +@lru_cache(maxsize=1) +def _android_downloads_folder() -> str: + """:return: downloads folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + downloads_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DOWNLOADS).getAbsolutePath() + except Exception: # noqa: BLE001 + downloads_dir = "/storage/emulated/0/Downloads" + + return downloads_dir + + +@lru_cache(maxsize=1) +def _android_pictures_folder() -> str: + """:return: pictures folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + pictures_dir: str = context.getExternalFilesDir(environment.DIRECTORY_PICTURES).getAbsolutePath() + except Exception: # noqa: BLE001 + pictures_dir = "/storage/emulated/0/Pictures" + + return pictures_dir + + +@lru_cache(maxsize=1) +def _android_videos_folder() -> str: + """:return: videos folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + videos_dir: str = context.getExternalFilesDir(environment.DIRECTORY_DCIM).getAbsolutePath() + except Exception: # noqa: BLE001 + videos_dir = "/storage/emulated/0/DCIM/Camera" + + return videos_dir + + +@lru_cache(maxsize=1) +def _android_music_folder() -> str: + """:return: music folder for the Android OS""" + # Get directories with pyjnius + try: + from jnius import autoclass + + context = autoclass("android.content.Context") + environment = autoclass("android.os.Environment") + music_dir: str = context.getExternalFilesDir(environment.DIRECTORY_MUSIC).getAbsolutePath() + except Exception: # noqa: BLE001 + music_dir = "/storage/emulated/0/Music" + + return music_dir + + __all__ = [ "Android", ] diff --git a/src/pip/_vendor/platformdirs/api.py b/src/pip/_vendor/platformdirs/api.py index f140e8b6d..d64ebb9d4 100644 --- a/src/pip/_vendor/platformdirs/api.py +++ b/src/pip/_vendor/platformdirs/api.py @@ -1,29 +1,33 @@ +"""Base API.""" from __future__ import annotations import os -import sys from abc import ABC, abstractmethod from pathlib import Path +from typing import TYPE_CHECKING -if sys.version_info >= (3, 8): # pragma: no branch - from typing import Literal # pragma: no cover +if TYPE_CHECKING: + import sys + + if sys.version_info >= (3, 8): # pragma: no cover (py38+) + from typing import Literal + else: # pragma: no cover (py38+) + from pip._vendor.typing_extensions import Literal class PlatformDirsABC(ABC): - """ - Abstract base class for platform directories. - """ + """Abstract base class for platform directories.""" - def __init__( + def __init__( # noqa: PLR0913 self, appname: str | None = None, appauthor: str | None | Literal[False] = None, version: str | None = None, - roaming: bool = False, - multipath: bool = False, - opinion: bool = True, - ensure_exists: bool = False, - ): + roaming: bool = False, # noqa: FBT001, FBT002 + multipath: bool = False, # noqa: FBT001, FBT002 + opinion: bool = True, # noqa: FBT001, FBT002 + ensure_exists: bool = False, # noqa: FBT001, FBT002 + ) -> None: """ Create a new platform directory. @@ -70,7 +74,7 @@ class PlatformDirsABC(ABC): params.append(self.appname) if self.version: params.append(self.version) - path = os.path.join(base[0], *params) + path = os.path.join(base[0], *params) # noqa: PTH118 self._optionally_create_directory(path) return path @@ -123,6 +127,26 @@ class PlatformDirsABC(ABC): def user_documents_dir(self) -> str: """:return: documents directory tied to the user""" + @property + @abstractmethod + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user""" + + @property + @abstractmethod + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user""" + + @property + @abstractmethod + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user""" + + @property + @abstractmethod + def user_music_dir(self) -> str: + """:return: music directory tied to the user""" + @property @abstractmethod def user_runtime_dir(self) -> str: @@ -173,6 +197,26 @@ class PlatformDirsABC(ABC): """:return: documents path tied to the user""" return Path(self.user_documents_dir) + @property + def user_downloads_path(self) -> Path: + """:return: downloads path tied to the user""" + return Path(self.user_downloads_dir) + + @property + def user_pictures_path(self) -> Path: + """:return: pictures path tied to the user""" + return Path(self.user_pictures_dir) + + @property + def user_videos_path(self) -> Path: + """:return: videos path tied to the user""" + return Path(self.user_videos_dir) + + @property + def user_music_path(self) -> Path: + """:return: music path tied to the user""" + return Path(self.user_music_dir) + @property def user_runtime_path(self) -> Path: """:return: runtime path tied to the user""" diff --git a/src/pip/_vendor/platformdirs/macos.py b/src/pip/_vendor/platformdirs/macos.py index ec9751129..a753e2a3a 100644 --- a/src/pip/_vendor/platformdirs/macos.py +++ b/src/pip/_vendor/platformdirs/macos.py @@ -1,6 +1,7 @@ +"""macOS.""" from __future__ import annotations -import os +import os.path from .api import PlatformDirsABC @@ -17,7 +18,7 @@ class MacOS(PlatformDirsABC): @property def user_data_dir(self) -> str: """:return: data directory tied to the user, e.g. ``~/Library/Application Support/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Application Support")) + return self._append_app_name_and_version(os.path.expanduser("~/Library/Application Support")) # noqa: PTH111 @property def site_data_dir(self) -> str: @@ -37,7 +38,7 @@ class MacOS(PlatformDirsABC): @property def user_cache_dir(self) -> str: """:return: cache directory tied to the user, e.g. ``~/Library/Caches/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches")) + return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches")) # noqa: PTH111 @property def site_cache_dir(self) -> str: @@ -52,17 +53,37 @@ class MacOS(PlatformDirsABC): @property def user_log_dir(self) -> str: """:return: log directory tied to the user, e.g. ``~/Library/Logs/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Logs")) + return self._append_app_name_and_version(os.path.expanduser("~/Library/Logs")) # noqa: PTH111 @property def user_documents_dir(self) -> str: """:return: documents directory tied to the user, e.g. ``~/Documents``""" - return os.path.expanduser("~/Documents") + return os.path.expanduser("~/Documents") # noqa: PTH111 + + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user, e.g. ``~/Downloads``""" + return os.path.expanduser("~/Downloads") # noqa: PTH111 + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user, e.g. ``~/Pictures``""" + return os.path.expanduser("~/Pictures") # noqa: PTH111 + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user, e.g. ``~/Movies``""" + return os.path.expanduser("~/Movies") # noqa: PTH111 + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user, e.g. ``~/Music``""" + return os.path.expanduser("~/Music") # noqa: PTH111 @property def user_runtime_dir(self) -> str: """:return: runtime directory tied to the user, e.g. ``~/Library/Caches/TemporaryItems/$appname/$version``""" - return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches/TemporaryItems")) + return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches/TemporaryItems")) # noqa: PTH111 __all__ = [ diff --git a/src/pip/_vendor/platformdirs/unix.py b/src/pip/_vendor/platformdirs/unix.py index 17d355da9..468b0ab49 100644 --- a/src/pip/_vendor/platformdirs/unix.py +++ b/src/pip/_vendor/platformdirs/unix.py @@ -1,3 +1,4 @@ +"""Unix.""" from __future__ import annotations import os @@ -7,12 +8,14 @@ from pathlib import Path from .api import PlatformDirsABC -if sys.platform.startswith("linux"): # pragma: no branch # no op check, only to please the type checker - from os import getuid -else: +if sys.platform == "win32": def getuid() -> int: - raise RuntimeError("should only be used on Linux") + msg = "should only be used on Unix" + raise RuntimeError(msg) + +else: + from os import getuid class Unix(PlatformDirsABC): @@ -36,7 +39,7 @@ class Unix(PlatformDirsABC): """ path = os.environ.get("XDG_DATA_HOME", "") if not path.strip(): - path = os.path.expanduser("~/.local/share") + path = os.path.expanduser("~/.local/share") # noqa: PTH111 return self._append_app_name_and_version(path) @property @@ -56,7 +59,7 @@ class Unix(PlatformDirsABC): path_list = path.split(os.pathsep) if not self.multipath: path_list = path_list[0:1] - path_list = [self._append_app_name_and_version(os.path.expanduser(p)) for p in path_list] + path_list = [self._append_app_name_and_version(os.path.expanduser(p)) for p in path_list] # noqa: PTH111 return os.pathsep.join(path_list) @property @@ -67,7 +70,7 @@ class Unix(PlatformDirsABC): """ path = os.environ.get("XDG_CONFIG_HOME", "") if not path.strip(): - path = os.path.expanduser("~/.config") + path = os.path.expanduser("~/.config") # noqa: PTH111 return self._append_app_name_and_version(path) @property @@ -91,15 +94,13 @@ class Unix(PlatformDirsABC): """ path = os.environ.get("XDG_CACHE_HOME", "") if not path.strip(): - path = os.path.expanduser("~/.cache") + path = os.path.expanduser("~/.cache") # noqa: PTH111 return self._append_app_name_and_version(path) @property def site_cache_dir(self) -> str: - """ - :return: cache directory shared by users, e.g. ``/var/tmp/$appname/$version`` - """ - return self._append_app_name_and_version("/var/tmp") + """:return: cache directory shared by users, e.g. ``/var/tmp/$appname/$version``""" + return self._append_app_name_and_version("/var/tmp") # noqa: S108 @property def user_state_dir(self) -> str: @@ -109,41 +110,60 @@ class Unix(PlatformDirsABC): """ path = os.environ.get("XDG_STATE_HOME", "") if not path.strip(): - path = os.path.expanduser("~/.local/state") + path = os.path.expanduser("~/.local/state") # noqa: PTH111 return self._append_app_name_and_version(path) @property def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_state_dir` if not opinionated else ``log`` in it - """ + """:return: log directory tied to the user, same as `user_state_dir` if not opinionated else ``log`` in it""" path = self.user_state_dir if self.opinion: - path = os.path.join(path, "log") + path = os.path.join(path, "log") # noqa: PTH118 return path @property def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user, e.g. ``~/Documents`` - """ - documents_dir = _get_user_dirs_folder("XDG_DOCUMENTS_DIR") - if documents_dir is None: - documents_dir = os.environ.get("XDG_DOCUMENTS_DIR", "").strip() - if not documents_dir: - documents_dir = os.path.expanduser("~/Documents") + """:return: documents directory tied to the user, e.g. ``~/Documents``""" + return _get_user_media_dir("XDG_DOCUMENTS_DIR", "~/Documents") - return documents_dir + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user, e.g. ``~/Downloads``""" + return _get_user_media_dir("XDG_DOWNLOAD_DIR", "~/Downloads") + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user, e.g. ``~/Pictures``""" + return _get_user_media_dir("XDG_PICTURES_DIR", "~/Pictures") + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user, e.g. ``~/Videos``""" + return _get_user_media_dir("XDG_VIDEOS_DIR", "~/Videos") + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user, e.g. ``~/Music``""" + return _get_user_media_dir("XDG_MUSIC_DIR", "~/Music") @property def user_runtime_dir(self) -> str: """ :return: runtime directory tied to the user, e.g. ``/run/user/$(id -u)/$appname/$version`` or - ``$XDG_RUNTIME_DIR/$appname/$version`` + ``$XDG_RUNTIME_DIR/$appname/$version``. + + For FreeBSD/OpenBSD/NetBSD, it would return ``/var/run/user/$(id -u)/$appname/$version`` if + exists, otherwise ``/tmp/runtime-$(id -u)/$appname/$version``, if``$XDG_RUNTIME_DIR`` + is not set. """ path = os.environ.get("XDG_RUNTIME_DIR", "") if not path.strip(): - path = f"/run/user/{getuid()}" + if sys.platform.startswith(("freebsd", "openbsd", "netbsd")): + path = f"/var/run/user/{getuid()}" + if not Path(path).exists(): + path = f"/tmp/runtime-{getuid()}" # noqa: S108 + else: + path = f"/run/user/{getuid()}" return self._append_app_name_and_version(path) @property @@ -168,13 +188,23 @@ class Unix(PlatformDirsABC): return Path(directory) +def _get_user_media_dir(env_var: str, fallback_tilde_path: str) -> str: + media_dir = _get_user_dirs_folder(env_var) + if media_dir is None: + media_dir = os.environ.get(env_var, "").strip() + if not media_dir: + media_dir = os.path.expanduser(fallback_tilde_path) # noqa: PTH111 + + return media_dir + + def _get_user_dirs_folder(key: str) -> str | None: - """Return directory from user-dirs.dirs config file. See https://freedesktop.org/wiki/Software/xdg-user-dirs/""" - user_dirs_config_path = os.path.join(Unix().user_config_dir, "user-dirs.dirs") - if os.path.exists(user_dirs_config_path): + """Return directory from user-dirs.dirs config file. See https://freedesktop.org/wiki/Software/xdg-user-dirs/.""" + user_dirs_config_path = Path(Unix().user_config_dir) / "user-dirs.dirs" + if user_dirs_config_path.exists(): parser = ConfigParser() - with open(user_dirs_config_path) as stream: + with user_dirs_config_path.open() as stream: # Add fake section header, so ConfigParser doesn't complain parser.read_string(f"[top]\n{stream.read()}") @@ -183,8 +213,7 @@ def _get_user_dirs_folder(key: str) -> str | None: path = parser["top"][key].strip('"') # Handle relative home paths - path = path.replace("$HOME", os.path.expanduser("~")) - return path + return path.replace("$HOME", os.path.expanduser("~")) # noqa: PTH111 return None diff --git a/src/pip/_vendor/platformdirs/version.py b/src/pip/_vendor/platformdirs/version.py index d906a2c99..dc8c44cf7 100644 --- a/src/pip/_vendor/platformdirs/version.py +++ b/src/pip/_vendor/platformdirs/version.py @@ -1,4 +1,4 @@ # file generated by setuptools_scm # don't change, don't track in version control -__version__ = version = '3.2.0' -__version_tuple__ = version_tuple = (3, 2, 0) +__version__ = version = '3.8.1' +__version_tuple__ = version_tuple = (3, 8, 1) diff --git a/src/pip/_vendor/platformdirs/windows.py b/src/pip/_vendor/platformdirs/windows.py index e7573c3d6..b52c9c6ea 100644 --- a/src/pip/_vendor/platformdirs/windows.py +++ b/src/pip/_vendor/platformdirs/windows.py @@ -1,16 +1,21 @@ +"""Windows.""" from __future__ import annotations import ctypes import os import sys from functools import lru_cache -from typing import Callable +from typing import TYPE_CHECKING from .api import PlatformDirsABC +if TYPE_CHECKING: + from collections.abc import Callable + class Windows(PlatformDirsABC): - """`MSDN on where to store app data files + """ + `MSDN on where to store app data files `_. Makes use of the `appname `, @@ -43,7 +48,7 @@ class Windows(PlatformDirsABC): params.append(opinion_value) if self.version: params.append(self.version) - path = os.path.join(path, *params) + path = os.path.join(path, *params) # noqa: PTH118 self._optionally_create_directory(path) return path @@ -85,36 +90,53 @@ class Windows(PlatformDirsABC): @property def user_log_dir(self) -> str: - """ - :return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``Logs`` in it - """ + """:return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``Logs`` in it""" path = self.user_data_dir if self.opinion: - path = os.path.join(path, "Logs") + path = os.path.join(path, "Logs") # noqa: PTH118 self._optionally_create_directory(path) return path @property def user_documents_dir(self) -> str: - """ - :return: documents directory tied to the user e.g. ``%USERPROFILE%\\Documents`` - """ + """:return: documents directory tied to the user e.g. ``%USERPROFILE%\\Documents``""" return os.path.normpath(get_win_folder("CSIDL_PERSONAL")) + @property + def user_downloads_dir(self) -> str: + """:return: downloads directory tied to the user e.g. ``%USERPROFILE%\\Downloads``""" + return os.path.normpath(get_win_folder("CSIDL_DOWNLOADS")) + + @property + def user_pictures_dir(self) -> str: + """:return: pictures directory tied to the user e.g. ``%USERPROFILE%\\Pictures``""" + return os.path.normpath(get_win_folder("CSIDL_MYPICTURES")) + + @property + def user_videos_dir(self) -> str: + """:return: videos directory tied to the user e.g. ``%USERPROFILE%\\Videos``""" + return os.path.normpath(get_win_folder("CSIDL_MYVIDEO")) + + @property + def user_music_dir(self) -> str: + """:return: music directory tied to the user e.g. ``%USERPROFILE%\\Music``""" + return os.path.normpath(get_win_folder("CSIDL_MYMUSIC")) + @property def user_runtime_dir(self) -> str: """ :return: runtime directory tied to the user, e.g. ``%USERPROFILE%\\AppData\\Local\\Temp\\$appauthor\\$appname`` """ - path = os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Temp")) + path = os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Temp")) # noqa: PTH118 return self._append_parts(path) def get_win_folder_from_env_vars(csidl_name: str) -> str: """Get folder from environment variables.""" - if csidl_name == "CSIDL_PERSONAL": # does not have an environment name - return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Documents") + result = get_win_folder_if_csidl_name_not_env_var(csidl_name) + if result is not None: + return result env_var_name = { "CSIDL_APPDATA": "APPDATA", @@ -122,28 +144,54 @@ def get_win_folder_from_env_vars(csidl_name: str) -> str: "CSIDL_LOCAL_APPDATA": "LOCALAPPDATA", }.get(csidl_name) if env_var_name is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) result = os.environ.get(env_var_name) if result is None: - raise ValueError(f"Unset environment variable: {env_var_name}") + msg = f"Unset environment variable: {env_var_name}" + raise ValueError(msg) return result -def get_win_folder_from_registry(csidl_name: str) -> str: - """Get folder from the registry. +def get_win_folder_if_csidl_name_not_env_var(csidl_name: str) -> str | None: + """Get folder for a CSIDL name that does not exist as an environment variable.""" + if csidl_name == "CSIDL_PERSONAL": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Documents") # noqa: PTH118 - This is a fallback technique at best. I'm not sure if using the - registry for this guarantees us the correct answer for all CSIDL_* - names. + if csidl_name == "CSIDL_DOWNLOADS": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Downloads") # noqa: PTH118 + + if csidl_name == "CSIDL_MYPICTURES": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Pictures") # noqa: PTH118 + + if csidl_name == "CSIDL_MYVIDEO": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Videos") # noqa: PTH118 + + if csidl_name == "CSIDL_MYMUSIC": + return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Music") # noqa: PTH118 + return None + + +def get_win_folder_from_registry(csidl_name: str) -> str: + """ + Get folder from the registry. + + This is a fallback technique at best. I'm not sure if using the registry for these guarantees us the correct answer + for all CSIDL_* names. """ shell_folder_name = { "CSIDL_APPDATA": "AppData", "CSIDL_COMMON_APPDATA": "Common AppData", "CSIDL_LOCAL_APPDATA": "Local AppData", "CSIDL_PERSONAL": "Personal", + "CSIDL_DOWNLOADS": "{374DE290-123F-4565-9164-39C4925E467B}", + "CSIDL_MYPICTURES": "My Pictures", + "CSIDL_MYVIDEO": "My Video", + "CSIDL_MYMUSIC": "My Music", }.get(csidl_name) if shell_folder_name is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) if sys.platform != "win32": # only needed for mypy type checker to know that this code runs only on Windows raise NotImplementedError import winreg @@ -155,25 +203,37 @@ def get_win_folder_from_registry(csidl_name: str) -> str: def get_win_folder_via_ctypes(csidl_name: str) -> str: """Get folder with ctypes.""" + # There is no 'CSIDL_DOWNLOADS'. + # Use 'CSIDL_PROFILE' (40) and append the default folder 'Downloads' instead. + # https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid + csidl_const = { "CSIDL_APPDATA": 26, "CSIDL_COMMON_APPDATA": 35, "CSIDL_LOCAL_APPDATA": 28, "CSIDL_PERSONAL": 5, + "CSIDL_MYPICTURES": 39, + "CSIDL_MYVIDEO": 14, + "CSIDL_MYMUSIC": 13, + "CSIDL_DOWNLOADS": 40, }.get(csidl_name) if csidl_const is None: - raise ValueError(f"Unknown CSIDL name: {csidl_name}") + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) buf = ctypes.create_unicode_buffer(1024) windll = getattr(ctypes, "windll") # noqa: B009 # using getattr to avoid false positive with mypy type checker windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) # Downgrade to short path name if it has highbit chars. - if any(ord(c) > 255 for c in buf): + if any(ord(c) > 255 for c in buf): # noqa: PLR2004 buf2 = ctypes.create_unicode_buffer(1024) if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): buf = buf2 + if csidl_name == "CSIDL_DOWNLOADS": + return os.path.join(buf.value, "Downloads") # noqa: PTH118 + return buf.value diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index dcf89dc04..07671fb58 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -4,7 +4,7 @@ distlib==0.3.6 distro==1.8.0 msgpack==1.0.5 packaging==21.3 -platformdirs==3.2.0 +platformdirs==3.8.1 pyparsing==3.0.9 pyproject-hooks==1.0.0 requests==2.31.0 From 6ee8884ac4a241f665aa712bcbe0e185012bccdb Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 9 Jul 2023 21:30:39 +0100 Subject: [PATCH 105/216] Upgrade pyparsing to 3.1.0 --- news/pyparsing.vendor.rst | 1 + src/pip/_vendor/pyparsing/__init__.py | 75 +- src/pip/_vendor/pyparsing/actions.py | 34 +- src/pip/_vendor/pyparsing/common.py | 58 +- src/pip/_vendor/pyparsing/core.py | 1193 +++++++++++------ src/pip/_vendor/pyparsing/diagram/__init__.py | 32 +- src/pip/_vendor/pyparsing/exceptions.py | 64 +- src/pip/_vendor/pyparsing/helpers.py | 196 +-- src/pip/_vendor/pyparsing/results.py | 128 +- src/pip/_vendor/pyparsing/testing.py | 24 +- src/pip/_vendor/pyparsing/unicode.py | 103 +- src/pip/_vendor/pyparsing/util.py | 89 +- src/pip/_vendor/vendor.txt | 2 +- 13 files changed, 1231 insertions(+), 768 deletions(-) create mode 100644 news/pyparsing.vendor.rst diff --git a/news/pyparsing.vendor.rst b/news/pyparsing.vendor.rst new file mode 100644 index 000000000..9feffb246 --- /dev/null +++ b/news/pyparsing.vendor.rst @@ -0,0 +1 @@ +Upgrade pyparsing to 3.1.0 diff --git a/src/pip/_vendor/pyparsing/__init__.py b/src/pip/_vendor/pyparsing/__init__.py index 75372500e..88bc10ac1 100644 --- a/src/pip/_vendor/pyparsing/__init__.py +++ b/src/pip/_vendor/pyparsing/__init__.py @@ -56,7 +56,7 @@ self-explanatory class names, and the use of :class:`'+'`, :class:`'|'`, :class:`'^'` and :class:`'&'` operators. The :class:`ParseResults` object returned from -:class:`ParserElement.parseString` can be +:class:`ParserElement.parse_string` can be accessed as a nested list, a dictionary, or an object with named attributes. @@ -85,11 +85,11 @@ classes inherit from. Use the docstrings for examples of how to: and :class:`'&'` operators to combine simple expressions into more complex ones - associate names with your parsed results using - :class:`ParserElement.setResultsName` + :class:`ParserElement.set_results_name` - access the parsed data, which is returned as a :class:`ParseResults` object - - find some helpful expression short-cuts like :class:`delimitedList` - and :class:`oneOf` + - find some helpful expression short-cuts like :class:`DelimitedList` + and :class:`one_of` - find more useful common expressions in the :class:`pyparsing_common` namespace class """ @@ -106,30 +106,22 @@ class version_info(NamedTuple): @property def __version__(self): return ( - "{}.{}.{}".format(self.major, self.minor, self.micro) + f"{self.major}.{self.minor}.{self.micro}" + ( - "{}{}{}".format( - "r" if self.releaselevel[0] == "c" else "", - self.releaselevel[0], - self.serial, - ), + f"{'r' if self.releaselevel[0] == 'c' else ''}{self.releaselevel[0]}{self.serial}", "", )[self.releaselevel == "final"] ) def __str__(self): - return "{} {} / {}".format(__name__, self.__version__, __version_time__) + return f"{__name__} {self.__version__} / {__version_time__}" def __repr__(self): - return "{}.{}({})".format( - __name__, - type(self).__name__, - ", ".join("{}={!r}".format(*nv) for nv in zip(self._fields, self)), - ) + return f"{__name__}.{type(self).__name__}({', '.join('{}={!r}'.format(*nv) for nv in zip(self._fields, self))})" -__version_info__ = version_info(3, 0, 9, "final", 0) -__version_time__ = "05 May 2022 07:02 UTC" +__version_info__ = version_info(3, 1, 0, "final", 1) +__version_time__ = "18 Jun 2023 14:05 UTC" __version__ = __version_info__.__version__ __versionTime__ = __version_time__ __author__ = "Paul McGuire " @@ -139,9 +131,9 @@ from .exceptions import * from .actions import * from .core import __diag__, __compat__ from .results import * -from .core import * +from .core import * # type: ignore[misc, assignment] from .core import _builtin_exprs as core_builtin_exprs -from .helpers import * +from .helpers import * # type: ignore[misc, assignment] from .helpers import _builtin_exprs as helper_builtin_exprs from .unicode import unicode_set, UnicodeRangeList, pyparsing_unicode as unicode @@ -153,11 +145,11 @@ from .common import ( # define backward compat synonyms if "pyparsing_unicode" not in globals(): - pyparsing_unicode = unicode + pyparsing_unicode = unicode # type: ignore[misc] if "pyparsing_common" not in globals(): - pyparsing_common = common + pyparsing_common = common # type: ignore[misc] if "pyparsing_test" not in globals(): - pyparsing_test = testing + pyparsing_test = testing # type: ignore[misc] core_builtin_exprs += common_builtin_exprs + helper_builtin_exprs @@ -174,7 +166,9 @@ __all__ = [ "CaselessKeyword", "CaselessLiteral", "CharsNotIn", + "CloseMatch", "Combine", + "DelimitedList", "Dict", "Each", "Empty", @@ -227,9 +221,11 @@ __all__ = [ "alphas8bit", "any_close_tag", "any_open_tag", + "autoname_elements", "c_style_comment", "col", "common_html_entity", + "condition_as_parse_action", "counted_array", "cpp_style_comment", "dbl_quoted_string", @@ -241,6 +237,7 @@ __all__ = [ "html_comment", "identchars", "identbodychars", + "infix_notation", "java_style_comment", "line", "line_end", @@ -255,8 +252,12 @@ __all__ = [ "null_debug_action", "nums", "one_of", + "original_text_for", "printables", "punc8bit", + "pyparsing_common", + "pyparsing_test", + "pyparsing_unicode", "python_style_comment", "quoted_string", "remove_quotes", @@ -267,28 +268,20 @@ __all__ = [ "srange", "string_end", "string_start", + "token_map", "trace_parse_action", + "ungroup", + "unicode_set", "unicode_string", "with_attribute", - "indentedBlock", - "original_text_for", - "ungroup", - "infix_notation", - "locatedExpr", "with_class", - "CloseMatch", - "token_map", - "pyparsing_common", - "pyparsing_unicode", - "unicode_set", - "condition_as_parse_action", - "pyparsing_test", # pre-PEP8 compatibility names "__versionTime__", "anyCloseTag", "anyOpenTag", "cStyleComment", "commonHTMLEntity", + "conditionAsParseAction", "countedArray", "cppStyleComment", "dblQuotedString", @@ -296,9 +289,12 @@ __all__ = [ "delimitedList", "dictOf", "htmlComment", + "indentedBlock", + "infixNotation", "javaStyleComment", "lineEnd", "lineStart", + "locatedExpr", "makeHTMLTags", "makeXMLTags", "matchOnlyAtCol", @@ -308,6 +304,7 @@ __all__ = [ "nullDebugAction", "oneOf", "opAssoc", + "originalTextFor", "pythonStyleComment", "quotedString", "removeQuotes", @@ -317,15 +314,9 @@ __all__ = [ "sglQuotedString", "stringEnd", "stringStart", + "tokenMap", "traceParseAction", "unicodeString", "withAttribute", - "indentedBlock", - "originalTextFor", - "infixNotation", - "locatedExpr", "withClass", - "tokenMap", - "conditionAsParseAction", - "autoname_elements", ] diff --git a/src/pip/_vendor/pyparsing/actions.py b/src/pip/_vendor/pyparsing/actions.py index f72c66e74..ca6e4c6af 100644 --- a/src/pip/_vendor/pyparsing/actions.py +++ b/src/pip/_vendor/pyparsing/actions.py @@ -1,7 +1,7 @@ # actions.py from .exceptions import ParseException -from .util import col +from .util import col, replaced_by_pep8 class OnlyOnce: @@ -38,7 +38,7 @@ def match_only_at_col(n): def verify_col(strg, locn, toks): if col(locn, strg) != n: - raise ParseException(strg, locn, "matched token not at column {}".format(n)) + raise ParseException(strg, locn, f"matched token not at column {n}") return verify_col @@ -148,15 +148,13 @@ def with_attribute(*args, **attr_dict): raise ParseException( s, l, - "attribute {!r} has value {!r}, must be {!r}".format( - attrName, tokens[attrName], attrValue - ), + f"attribute {attrName!r} has value {tokens[attrName]!r}, must be {attrValue!r}", ) return pa -with_attribute.ANY_VALUE = object() +with_attribute.ANY_VALUE = object() # type: ignore [attr-defined] def with_class(classname, namespace=""): @@ -195,13 +193,25 @@ def with_class(classname, namespace=""): 1 4 0 1 0 1,3 2,3 1,1 """ - classattr = "{}:class".format(namespace) if namespace else "class" + classattr = f"{namespace}:class" if namespace else "class" return with_attribute(**{classattr: classname}) # pre-PEP8 compatibility symbols -replaceWith = replace_with -removeQuotes = remove_quotes -withAttribute = with_attribute -withClass = with_class -matchOnlyAtCol = match_only_at_col +# fmt: off +@replaced_by_pep8(replace_with) +def replaceWith(): ... + +@replaced_by_pep8(remove_quotes) +def removeQuotes(): ... + +@replaced_by_pep8(with_attribute) +def withAttribute(): ... + +@replaced_by_pep8(with_class) +def withClass(): ... + +@replaced_by_pep8(match_only_at_col) +def matchOnlyAtCol(): ... + +# fmt: on diff --git a/src/pip/_vendor/pyparsing/common.py b/src/pip/_vendor/pyparsing/common.py index 1859fb79c..7a666b276 100644 --- a/src/pip/_vendor/pyparsing/common.py +++ b/src/pip/_vendor/pyparsing/common.py @@ -1,6 +1,6 @@ # common.py from .core import * -from .helpers import delimited_list, any_open_tag, any_close_tag +from .helpers import DelimitedList, any_open_tag, any_close_tag from datetime import datetime @@ -22,17 +22,17 @@ class pyparsing_common: Parse actions: - - :class:`convertToInteger` - - :class:`convertToFloat` - - :class:`convertToDate` - - :class:`convertToDatetime` - - :class:`stripHTMLTags` - - :class:`upcaseTokens` - - :class:`downcaseTokens` + - :class:`convert_to_integer` + - :class:`convert_to_float` + - :class:`convert_to_date` + - :class:`convert_to_datetime` + - :class:`strip_html_tags` + - :class:`upcase_tokens` + - :class:`downcase_tokens` Example:: - pyparsing_common.number.runTests(''' + pyparsing_common.number.run_tests(''' # any int or real number, returned as the appropriate type 100 -100 @@ -42,7 +42,7 @@ class pyparsing_common: 1e-12 ''') - pyparsing_common.fnumber.runTests(''' + pyparsing_common.fnumber.run_tests(''' # any int or real number, returned as float 100 -100 @@ -52,19 +52,19 @@ class pyparsing_common: 1e-12 ''') - pyparsing_common.hex_integer.runTests(''' + pyparsing_common.hex_integer.run_tests(''' # hex numbers 100 FF ''') - pyparsing_common.fraction.runTests(''' + pyparsing_common.fraction.run_tests(''' # fractions 1/2 -3/4 ''') - pyparsing_common.mixed_integer.runTests(''' + pyparsing_common.mixed_integer.run_tests(''' # mixed fractions 1 1/2 @@ -73,8 +73,8 @@ class pyparsing_common: ''') import uuid - pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID)) - pyparsing_common.uuid.runTests(''' + pyparsing_common.uuid.set_parse_action(token_map(uuid.UUID)) + pyparsing_common.uuid.run_tests(''' # uuid 12345678-1234-5678-1234-567812345678 ''') @@ -260,8 +260,8 @@ class pyparsing_common: Example:: date_expr = pyparsing_common.iso8601_date.copy() - date_expr.setParseAction(pyparsing_common.convertToDate()) - print(date_expr.parseString("1999-12-31")) + date_expr.set_parse_action(pyparsing_common.convert_to_date()) + print(date_expr.parse_string("1999-12-31")) prints:: @@ -287,8 +287,8 @@ class pyparsing_common: Example:: dt_expr = pyparsing_common.iso8601_datetime.copy() - dt_expr.setParseAction(pyparsing_common.convertToDatetime()) - print(dt_expr.parseString("1999-12-31T23:59:59.999")) + dt_expr.set_parse_action(pyparsing_common.convert_to_datetime()) + print(dt_expr.parse_string("1999-12-31T23:59:59.999")) prints:: @@ -326,9 +326,9 @@ class pyparsing_common: # strip HTML links from normal text text = 'More info at the pyparsing wiki page' - td, td_end = makeHTMLTags("TD") - table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - print(table_text.parseString(text).body) + td, td_end = make_html_tags("TD") + table_text = td + SkipTo(td_end).set_parse_action(pyparsing_common.strip_html_tags)("body") + td_end + print(table_text.parse_string(text).body) Prints:: @@ -348,7 +348,7 @@ class pyparsing_common: .streamline() .set_name("commaItem") ) - comma_separated_list = delimited_list( + comma_separated_list = DelimitedList( Opt(quoted_string.copy() | _commasepitem, default="") ).set_name("comma separated list") """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" @@ -363,7 +363,7 @@ class pyparsing_common: url = Regex( # https://mathiasbynens.be/demo/url-regex # https://gist.github.com/dperini/729294 - r"^" + + r"(?P" + # protocol identifier (optional) # short syntax // still required r"(?:(?:(?Phttps?|ftp):)?\/\/)" + @@ -405,18 +405,26 @@ class pyparsing_common: r"(\?(?P[^#]*))?" + # fragment (optional) r"(#(?P\S*))?" + - r"$" + r")" ).set_name("url") + """URL (http/https/ftp scheme)""" # fmt: on # pre-PEP8 compatibility names convertToInteger = convert_to_integer + """Deprecated - use :class:`convert_to_integer`""" convertToFloat = convert_to_float + """Deprecated - use :class:`convert_to_float`""" convertToDate = convert_to_date + """Deprecated - use :class:`convert_to_date`""" convertToDatetime = convert_to_datetime + """Deprecated - use :class:`convert_to_datetime`""" stripHTMLTags = strip_html_tags + """Deprecated - use :class:`strip_html_tags`""" upcaseTokens = upcase_tokens + """Deprecated - use :class:`upcase_tokens`""" downcaseTokens = downcase_tokens + """Deprecated - use :class:`downcase_tokens`""" _builtin_exprs = [ diff --git a/src/pip/_vendor/pyparsing/core.py b/src/pip/_vendor/pyparsing/core.py index 6ff3c766f..8d5a856ec 100644 --- a/src/pip/_vendor/pyparsing/core.py +++ b/src/pip/_vendor/pyparsing/core.py @@ -1,19 +1,22 @@ # # core.py # + +from collections import deque import os import typing from typing import ( - NamedTuple, - Union, - Callable, Any, + Callable, Generator, - Tuple, List, - TextIO, - Set, + NamedTuple, Sequence, + Set, + TextIO, + Tuple, + Union, + cast, ) from abc import ABC, abstractmethod from enum import Enum @@ -40,6 +43,7 @@ from .util import ( _flatten, LRUMemo as _LRUMemo, UnboundedMemo as _UnboundedMemo, + replaced_by_pep8, ) from .exceptions import * from .actions import * @@ -134,6 +138,7 @@ class __diag__(__config_flags): class Diagnostics(Enum): """ Diagnostic configuration (all default to disabled) + - ``warn_multiple_tokens_in_named_alternation`` - flag to enable warnings when a results name is defined on a :class:`MatchFirst` or :class:`Or` expression with one or more :class:`And` subexpressions - ``warn_ungrouped_named_tokens_in_collection`` - flag to enable warnings when a results @@ -228,6 +233,8 @@ _single_arg_builtins = { } _generatorType = types.GeneratorType +ParseImplReturnType = Tuple[int, Any] +PostParseReturnType = Union[ParseResults, Sequence[ParseResults]] ParseAction = Union[ Callable[[], Any], Callable[[ParseResults], Any], @@ -256,7 +263,7 @@ hexnums = nums + "ABCDEFabcdef" alphanums = alphas + nums printables = "".join([c for c in string.printable if c not in string.whitespace]) -_trim_arity_call_line: traceback.StackSummary = None +_trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment] def _trim_arity(func, max_limit=3): @@ -269,11 +276,6 @@ def _trim_arity(func, max_limit=3): limit = 0 found_arity = False - def extract_tb(tb, limit=0): - frames = traceback.extract_tb(tb, limit=limit) - frame_summary = frames[-1] - return [frame_summary[:2]] - # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time @@ -297,8 +299,10 @@ def _trim_arity(func, max_limit=3): raise else: tb = te.__traceback__ + frames = traceback.extract_tb(tb, limit=2) + frame_summary = frames[-1] trim_arity_type_error = ( - extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth + [frame_summary[:2]][-1][:2] == pa_call_line_synth ) del tb @@ -320,7 +324,7 @@ def _trim_arity(func, max_limit=3): def condition_as_parse_action( - fn: ParseCondition, message: str = None, fatal: bool = False + fn: ParseCondition, message: typing.Optional[str] = None, fatal: bool = False ) -> ParseAction: """ Function to convert a simple predicate function that returns ``True`` or ``False`` @@ -353,15 +357,9 @@ def _default_start_debug_action( cache_hit_str = "*" if cache_hit else "" print( ( - "{}Match {} at loc {}({},{})\n {}\n {}^".format( - cache_hit_str, - expr, - loc, - lineno(loc, instring), - col(loc, instring), - line(loc, instring), - " " * (col(loc, instring) - 1), - ) + f"{cache_hit_str}Match {expr} at loc {loc}({lineno(loc, instring)},{col(loc, instring)})\n" + f" {line(loc, instring)}\n" + f" {' ' * (col(loc, instring) - 1)}^" ) ) @@ -375,7 +373,7 @@ def _default_success_debug_action( cache_hit: bool = False, ): cache_hit_str = "*" if cache_hit else "" - print("{}Matched {} -> {}".format(cache_hit_str, expr, toks.as_list())) + print(f"{cache_hit_str}Matched {expr} -> {toks.as_list()}") def _default_exception_debug_action( @@ -386,11 +384,7 @@ def _default_exception_debug_action( cache_hit: bool = False, ): cache_hit_str = "*" if cache_hit else "" - print( - "{}Match {} failed, {} raised: {}".format( - cache_hit_str, expr, type(exc).__name__, exc - ) - ) + print(f"{cache_hit_str}Match {expr} failed, {type(exc).__name__} raised: {exc}") def null_debug_action(*args): @@ -402,7 +396,7 @@ class ParserElement(ABC): DEFAULT_WHITE_CHARS: str = " \n\t\r" verbose_stacktrace: bool = False - _literalStringClass: typing.Optional[type] = None + _literalStringClass: type = None # type: ignore[assignment] @staticmethod def set_default_whitespace_chars(chars: str) -> None: @@ -447,6 +441,18 @@ class ParserElement(ABC): """ ParserElement._literalStringClass = cls + @classmethod + def using_each(cls, seq, **class_kwargs): + """ + Yields a sequence of class(obj, **class_kwargs) for obj in seq. + + Example:: + + LPAR, RPAR, LBRACE, RBRACE, SEMI = Suppress.using_each("(){};") + + """ + yield from (cls(obj, **class_kwargs) for obj in seq) + class DebugActions(NamedTuple): debug_try: typing.Optional[DebugStartAction] debug_match: typing.Optional[DebugSuccessAction] @@ -455,9 +461,9 @@ class ParserElement(ABC): def __init__(self, savelist: bool = False): self.parseAction: List[ParseAction] = list() self.failAction: typing.Optional[ParseFailAction] = None - self.customName = None - self._defaultName = None - self.resultsName = None + self.customName: str = None # type: ignore[assignment] + self._defaultName: typing.Optional[str] = None + self.resultsName: str = None # type: ignore[assignment] self.saveAsList = savelist self.skipWhitespace = True self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) @@ -490,12 +496,29 @@ class ParserElement(ABC): base.suppress_warning(Diagnostics.warn_on_parse_using_empty_Forward) # statement would normally raise a warning, but is now suppressed - print(base.parseString("x")) + print(base.parse_string("x")) """ self.suppress_warnings_.append(warning_type) return self + def visit_all(self): + """General-purpose method to yield all expressions and sub-expressions + in a grammar. Typically just for internal use. + """ + to_visit = deque([self]) + seen = set() + while to_visit: + cur = to_visit.popleft() + + # guard against looping forever through recursive grammars + if cur in seen: + continue + seen.add(cur) + + to_visit.extend(cur.recurse()) + yield cur + def copy(self) -> "ParserElement": """ Make a copy of this :class:`ParserElement`. Useful for defining @@ -585,11 +608,11 @@ class ParserElement(ABC): pdb.set_trace() return _parseMethod(instring, loc, doActions, callPreParse) - breaker._originalParseMethod = _parseMethod - self._parse = breaker + breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined] + self._parse = breaker # type: ignore [assignment] else: if hasattr(self._parse, "_originalParseMethod"): - self._parse = self._parse._originalParseMethod + self._parse = self._parse._originalParseMethod # type: ignore [attr-defined, assignment] return self def set_parse_action(self, *fns: ParseAction, **kwargs) -> "ParserElement": @@ -601,9 +624,9 @@ class ParserElement(ABC): Each parse action ``fn`` is a callable method with 0-3 arguments, called as ``fn(s, loc, toks)`` , ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object + - ``s`` = the original string being parsed (see note below) + - ``loc`` = the location of the matching substring + - ``toks`` = a list of the matched tokens, packaged as a :class:`ParseResults` object The parsed tokens are passed to the parse action as ParseResults. They can be modified in place using list-style append, extend, and pop operations to update @@ -621,7 +644,7 @@ class ParserElement(ABC): Optional keyword arguments: - - call_during_try = (default= ``False``) indicate if parse action should be run during + - ``call_during_try`` = (default= ``False``) indicate if parse action should be run during lookaheads and alternate testing. For parse actions that have side effects, it is important to only call the parse action once it is determined that it is being called as part of a successful parse. For parse actions that perform additional @@ -697,10 +720,10 @@ class ParserElement(ABC): Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise + - ``message`` = define a custom message to be used in the raised exception + - ``fatal`` = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - - call_during_try = boolean to indicate if this method should be called during internal tryParse calls, + - ``call_during_try`` = boolean to indicate if this method should be called during internal tryParse calls, default=False Example:: @@ -716,7 +739,9 @@ class ParserElement(ABC): for fn in fns: self.parseAction.append( condition_as_parse_action( - fn, message=kwargs.get("message"), fatal=kwargs.get("fatal", False) + fn, + message=str(kwargs.get("message")), + fatal=bool(kwargs.get("fatal", False)), ) ) @@ -731,30 +756,33 @@ class ParserElement(ABC): Fail acton fn is a callable function that takes the arguments ``fn(s, loc, expr, err)`` where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown + - ``s`` = string being parsed + - ``loc`` = location where expression match was attempted and failed + - ``expr`` = the parse expression that failed + - ``err`` = the exception thrown The function returns no value. It may throw :class:`ParseFatalException` if it is desired to stop parsing immediately.""" self.failAction = fn return self - def _skipIgnorables(self, instring, loc): + def _skipIgnorables(self, instring: str, loc: int) -> int: + if not self.ignoreExprs: + return loc exprsFound = True + ignore_expr_fns = [e._parse for e in self.ignoreExprs] while exprsFound: exprsFound = False - for e in self.ignoreExprs: + for ignore_fn in ignore_expr_fns: try: while 1: - loc, dummy = e._parse(instring, loc) + loc, dummy = ignore_fn(instring, loc) exprsFound = True except ParseException: pass return loc - def preParse(self, instring, loc): + def preParse(self, instring: str, loc: int) -> int: if self.ignoreExprs: loc = self._skipIgnorables(instring, loc) @@ -830,7 +858,7 @@ class ParserElement(ABC): try: for fn in self.parseAction: try: - tokens = fn(instring, tokens_start, ret_tokens) + tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type] except IndexError as parse_action_exc: exc = ParseException("exception raised in parse action") raise exc from parse_action_exc @@ -853,7 +881,7 @@ class ParserElement(ABC): else: for fn in self.parseAction: try: - tokens = fn(instring, tokens_start, ret_tokens) + tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type] except IndexError as parse_action_exc: exc = ParseException("exception raised in parse action") raise exc from parse_action_exc @@ -875,17 +903,24 @@ class ParserElement(ABC): return loc, ret_tokens - def try_parse(self, instring: str, loc: int, raise_fatal: bool = False) -> int: + def try_parse( + self, + instring: str, + loc: int, + *, + raise_fatal: bool = False, + do_actions: bool = False, + ) -> int: try: - return self._parse(instring, loc, doActions=False)[0] + return self._parse(instring, loc, doActions=do_actions)[0] except ParseFatalException: if raise_fatal: raise raise ParseException(instring, loc, self.errmsg, self) - def can_parse_next(self, instring: str, loc: int) -> bool: + def can_parse_next(self, instring: str, loc: int, do_actions: bool = False) -> bool: try: - self.try_parse(instring, loc) + self.try_parse(instring, loc, do_actions=do_actions) except (ParseException, IndexError): return False else: @@ -897,10 +932,23 @@ class ParserElement(ABC): Tuple[int, "Forward", bool], Tuple[int, Union[ParseResults, Exception]] ] = {} + class _CacheType(dict): + """ + class to help type checking + """ + + not_in_cache: bool + + def get(self, *args): + ... + + def set(self, *args): + ... + # argument cache for optimizing repeated calls when backtracking through recursive expressions packrat_cache = ( - {} - ) # this is set later by enabled_packrat(); this is here so that reset_cache() doesn't fail + _CacheType() + ) # set later by enable_packrat(); this is here so that reset_cache() doesn't fail packrat_cache_lock = RLock() packrat_cache_stats = [0, 0] @@ -930,24 +978,25 @@ class ParserElement(ABC): ParserElement.packrat_cache_stats[HIT] += 1 if self.debug and self.debugActions.debug_try: try: - self.debugActions.debug_try(instring, loc, self, cache_hit=True) + self.debugActions.debug_try(instring, loc, self, cache_hit=True) # type: ignore [call-arg] except TypeError: pass if isinstance(value, Exception): if self.debug and self.debugActions.debug_fail: try: self.debugActions.debug_fail( - instring, loc, self, value, cache_hit=True + instring, loc, self, value, cache_hit=True # type: ignore [call-arg] ) except TypeError: pass raise value + value = cast(Tuple[int, ParseResults, int], value) loc_, result, endloc = value[0], value[1].copy(), value[2] if self.debug and self.debugActions.debug_match: try: self.debugActions.debug_match( - instring, loc_, endloc, self, result, cache_hit=True + instring, loc_, endloc, self, result, cache_hit=True # type: ignore [call-arg] ) except TypeError: pass @@ -1009,7 +1058,7 @@ class ParserElement(ABC): Parameters: - - cache_size_limit - (default=``None``) - memoize at most this many + - ``cache_size_limit`` - (default=``None``) - memoize at most this many ``Forward`` elements during matching; if ``None`` (the default), memoize all ``Forward`` elements. @@ -1022,9 +1071,9 @@ class ParserElement(ABC): elif ParserElement._packratEnabled: raise RuntimeError("Packrat and Bounded Recursion are not compatible") if cache_size_limit is None: - ParserElement.recursion_memos = _UnboundedMemo() + ParserElement.recursion_memos = _UnboundedMemo() # type: ignore[assignment] elif cache_size_limit > 0: - ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) + ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment] else: raise NotImplementedError("Memo size of %s" % cache_size_limit) ParserElement._left_recursion_enabled = True @@ -1040,7 +1089,7 @@ class ParserElement(ABC): Parameters: - - cache_size_limit - (default= ``128``) - if an integer value is provided + - ``cache_size_limit`` - (default= ``128``) - if an integer value is provided will limit the size of the packrat cache; if None is passed, then the cache size will be unbounded; if 0 is passed, the cache will be effectively disabled. @@ -1070,7 +1119,7 @@ class ParserElement(ABC): if cache_size_limit is None: ParserElement.packrat_cache = _UnboundedCache() else: - ParserElement.packrat_cache = _FifoCache(cache_size_limit) + ParserElement.packrat_cache = _FifoCache(cache_size_limit) # type: ignore[assignment] ParserElement._parse = ParserElement._parseCache def parse_string( @@ -1088,7 +1137,7 @@ class ParserElement(ABC): an object with attributes if the given parser includes results names. If the input string is required to match the entire grammar, ``parse_all`` flag must be set to ``True``. This - is also equivalent to ending the grammar with :class:`StringEnd`(). + is also equivalent to ending the grammar with :class:`StringEnd`\\ (). To report proper column numbers, ``parse_string`` operates on a copy of the input string where all tabs are converted to spaces (8 spaces per tab, as per the default in ``string.expandtabs``). If the input string @@ -1198,7 +1247,9 @@ class ParserElement(ABC): try: while loc <= instrlen and matches < maxMatches: try: - preloc = preparseFn(instring, loc) + preloc: int = preparseFn(instring, loc) + nextLoc: int + tokens: ParseResults nextLoc, tokens = parseFn(instring, preloc, callPreParse=False) except ParseException: loc = preloc + 1 @@ -1352,7 +1403,7 @@ class ParserElement(ABC): def __add__(self, other) -> "ParserElement": """ Implementation of ``+`` operator - returns :class:`And`. Adding strings to a :class:`ParserElement` - converts them to :class:`Literal`s by default. + converts them to :class:`Literal`\\ s by default. Example:: @@ -1364,11 +1415,11 @@ class ParserElement(ABC): Hello, World! -> ['Hello', ',', 'World', '!'] - ``...`` may be used as a parse expression as a short form of :class:`SkipTo`. + ``...`` may be used as a parse expression as a short form of :class:`SkipTo`:: Literal('start') + ... + Literal('end') - is equivalent to: + is equivalent to:: Literal('start') + SkipTo('end')("_skipped*") + Literal('end') @@ -1382,11 +1433,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return And([self, other]) def __radd__(self, other) -> "ParserElement": @@ -1399,11 +1446,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other + self def __sub__(self, other) -> "ParserElement": @@ -1413,11 +1456,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return self + And._ErrorStop() + other def __rsub__(self, other) -> "ParserElement": @@ -1427,11 +1466,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other - self def __mul__(self, other) -> "ParserElement": @@ -1440,11 +1475,12 @@ class ParserElement(ABC): ``expr + expr + expr``. Expressions may also be multiplied by a 2-integer tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples may also include ``None`` as in: + - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent - to ``expr*n + ZeroOrMore(expr)`` - (read as "at least n instances of ``expr``") + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") - ``expr*(None, n)`` is equivalent to ``expr*(0, n)`` - (read as "0 to n instances of ``expr``") + (read as "0 to n instances of ``expr``") - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)`` - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)`` @@ -1477,17 +1513,9 @@ class ParserElement(ABC): minElements, optElements = other optElements -= minElements else: - raise TypeError( - "cannot multiply ParserElement and ({}) objects".format( - ",".join(type(item).__name__ for item in other) - ) - ) + return NotImplemented else: - raise TypeError( - "cannot multiply ParserElement and {} objects".format( - type(other).__name__ - ) - ) + return NotImplemented if minElements < 0: raise ValueError("cannot multiply ParserElement by negative value") @@ -1531,13 +1559,12 @@ class ParserElement(ABC): return _PendingSkip(self, must_skip=True) if isinstance(other, str_type): + # `expr | ""` is equivalent to `Opt(expr)` + if other == "": + return Opt(self) other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return MatchFirst([self, other]) def __ror__(self, other) -> "ParserElement": @@ -1547,11 +1574,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other | self def __xor__(self, other) -> "ParserElement": @@ -1561,11 +1584,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return Or([self, other]) def __rxor__(self, other) -> "ParserElement": @@ -1575,11 +1594,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other ^ self def __and__(self, other) -> "ParserElement": @@ -1589,11 +1604,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return Each([self, other]) def __rand__(self, other) -> "ParserElement": @@ -1603,11 +1614,7 @@ class ParserElement(ABC): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - raise TypeError( - "Cannot combine element of type {} with ParserElement".format( - type(other).__name__ - ) - ) + return NotImplemented return other & self def __invert__(self) -> "ParserElement": @@ -1636,38 +1643,58 @@ class ParserElement(ABC): ``None`` may be used in place of ``...``. - Note that ``expr[..., n]`` and ``expr[m, n]``do not raise an exception - if more than ``n`` ``expr``s exist in the input stream. If this behavior is + Note that ``expr[..., n]`` and ``expr[m, n]`` do not raise an exception + if more than ``n`` ``expr``\\ s exist in the input stream. If this behavior is desired, then write ``expr[..., n] + ~expr``. + + For repetition with a stop_on expression, use slice notation: + + - ``expr[...: end_expr]`` and ``expr[0, ...: end_expr]`` are equivalent to ``ZeroOrMore(expr, stop_on=end_expr)`` + - ``expr[1, ...: end_expr]`` is equivalent to ``OneOrMore(expr, stop_on=end_expr)`` + """ + stop_on_defined = False + stop_on = NoMatch() + if isinstance(key, slice): + key, stop_on = key.start, key.stop + if key is None: + key = ... + stop_on_defined = True + elif isinstance(key, tuple) and isinstance(key[-1], slice): + key, stop_on = (key[0], key[1].start), key[1].stop + stop_on_defined = True + # convert single arg keys to tuples + if isinstance(key, str_type): + key = (key,) try: - if isinstance(key, str_type): - key = (key,) iter(key) except TypeError: key = (key, key) if len(key) > 2: raise TypeError( - "only 1 or 2 index arguments supported ({}{})".format( - key[:5], "... [{}]".format(len(key)) if len(key) > 5 else "" - ) + f"only 1 or 2 index arguments supported ({key[:5]}{f'... [{len(key)}]' if len(key) > 5 else ''})" ) # clip to 2 elements ret = self * tuple(key[:2]) + ret = typing.cast(_MultipleMatch, ret) + + if stop_on_defined: + ret.stopOn(stop_on) + return ret - def __call__(self, name: str = None) -> "ParserElement": + def __call__(self, name: typing.Optional[str] = None) -> "ParserElement": """ Shortcut for :class:`set_results_name`, with ``list_all_matches=False``. If ``name`` is given with a trailing ``'*'`` character, then ``list_all_matches`` will be passed as ``True``. - If ``name` is omitted, same as calling :class:`copy`. + If ``name`` is omitted, same as calling :class:`copy`. Example:: @@ -1775,17 +1802,18 @@ class ParserElement(ABC): should have the signature ``fn(input_string: str, location: int, expression: ParserElement, exception: Exception, cache_hit: bool)`` """ self.debugActions = self.DebugActions( - start_action or _default_start_debug_action, - success_action or _default_success_debug_action, - exception_action or _default_exception_debug_action, + start_action or _default_start_debug_action, # type: ignore[truthy-function] + success_action or _default_success_debug_action, # type: ignore[truthy-function] + exception_action or _default_exception_debug_action, # type: ignore[truthy-function] ) self.debug = True return self - def set_debug(self, flag: bool = True) -> "ParserElement": + def set_debug(self, flag: bool = True, recurse: bool = False) -> "ParserElement": """ Enable display of debugging messages while doing pattern matching. Set ``flag`` to ``True`` to enable, ``False`` to disable. + Set ``recurse`` to ``True`` to set the debug flag on this expression and all sub-expressions. Example:: @@ -1819,6 +1847,11 @@ class ParserElement(ABC): which makes debugging and exception messages easier to understand - for instance, the default name created for the :class:`Word` expression without calling ``set_name`` is ``"W:(A-Za-z)"``. """ + if recurse: + for expr in self.visit_all(): + expr.set_debug(flag, recurse=False) + return self + if flag: self.set_debug_actions( _default_start_debug_action, @@ -1836,7 +1869,7 @@ class ParserElement(ABC): return self._defaultName @abstractmethod - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: """ Child classes must define this method, which defines how the ``default_name`` is set. """ @@ -1844,7 +1877,9 @@ class ParserElement(ABC): def set_name(self, name: str) -> "ParserElement": """ Define name for this expression, makes debugging and exception messages clearer. + Example:: + Word(nums).parse_string("ABC") # -> Exception: Expected W:(0-9) (at char 0), (line:1, col:1) Word(nums).set_name("integer").parse_string("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ @@ -1870,7 +1905,7 @@ class ParserElement(ABC): self._defaultName = None return self - def recurse(self) -> Sequence["ParserElement"]: + def recurse(self) -> List["ParserElement"]: return [] def _checkRecursion(self, parseElementList): @@ -1882,6 +1917,11 @@ class ParserElement(ABC): """ Check defined expressions for valid structure, check for infinite recursive definitions. """ + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + DeprecationWarning, + stacklevel=2, + ) self._checkRecursion([]) def parse_file( @@ -1899,8 +1939,10 @@ class ParserElement(ABC): """ parseAll = parseAll or parse_all try: + file_or_filename = typing.cast(TextIO, file_or_filename) file_contents = file_or_filename.read() except AttributeError: + file_or_filename = typing.cast(str, file_or_filename) with open(file_or_filename, "r", encoding=encoding) as f: file_contents = f.read() try: @@ -1932,6 +1974,7 @@ class ParserElement(ABC): inline microtests of sub expressions while building up larger parser. Parameters: + - ``test_string`` - to test against this expression for a match - ``parse_all`` - (default= ``True``) - flag to pass to :class:`parse_string` when running tests @@ -1955,7 +1998,7 @@ class ParserElement(ABC): full_dump: bool = True, print_results: bool = True, failure_tests: bool = False, - post_parse: Callable[[str, ParseResults], str] = None, + post_parse: typing.Optional[Callable[[str, ParseResults], str]] = None, file: typing.Optional[TextIO] = None, with_line_numbers: bool = False, *, @@ -1963,7 +2006,7 @@ class ParserElement(ABC): fullDump: bool = True, printResults: bool = True, failureTests: bool = False, - postParse: Callable[[str, ParseResults], str] = None, + postParse: typing.Optional[Callable[[str, ParseResults], str]] = None, ) -> Tuple[bool, List[Tuple[str, Union[ParseResults, Exception]]]]: """ Execute the parse expression on a series of test strings, showing each @@ -1971,6 +2014,7 @@ class ParserElement(ABC): run a parse expression against a list of sample strings. Parameters: + - ``tests`` - a list of separate test strings, or a multiline string of test strings - ``parse_all`` - (default= ``True``) - flag to pass to :class:`parse_string` when running tests - ``comment`` - (default= ``'#'``) - expression for indicating embedded comments in the test @@ -2067,22 +2111,27 @@ class ParserElement(ABC): failureTests = failureTests or failure_tests postParse = postParse or post_parse if isinstance(tests, str_type): + tests = typing.cast(str, tests) line_strip = type(tests).strip tests = [line_strip(test_line) for test_line in tests.rstrip().splitlines()] - if isinstance(comment, str_type): - comment = Literal(comment) + comment_specified = comment is not None + if comment_specified: + if isinstance(comment, str_type): + comment = typing.cast(str, comment) + comment = Literal(comment) + comment = typing.cast(ParserElement, comment) if file is None: file = sys.stdout print_ = file.write result: Union[ParseResults, Exception] - allResults = [] - comments = [] + allResults: List[Tuple[str, Union[ParseResults, Exception]]] = [] + comments: List[str] = [] success = True NL = Literal(r"\n").add_parse_action(replace_with("\n")).ignore(quoted_string) BOM = "\ufeff" for t in tests: - if comment is not None and comment.matches(t, False) or comments and not t: + if comment_specified and comment.matches(t, False) or comments and not t: comments.append( pyparsing_test.with_line_numbers(t) if with_line_numbers else t ) @@ -2107,7 +2156,7 @@ class ParserElement(ABC): success = success and failureTests result = pe except Exception as exc: - out.append("FAIL-EXCEPTION: {}: {}".format(type(exc).__name__, exc)) + out.append(f"FAIL-EXCEPTION: {type(exc).__name__}: {exc}") if ParserElement.verbose_stacktrace: out.extend(traceback.format_tb(exc.__traceback__)) success = success and failureTests @@ -2127,9 +2176,7 @@ class ParserElement(ABC): except Exception as e: out.append(result.dump(full=fullDump)) out.append( - "{} failed: {}: {}".format( - postParse.__name__, type(e).__name__, e - ) + f"{postParse.__name__} failed: {type(e).__name__}: {e}" ) else: out.append(result.dump(full=fullDump)) @@ -2148,19 +2195,28 @@ class ParserElement(ABC): vertical: int = 3, show_results_names: bool = False, show_groups: bool = False, + embed: bool = False, **kwargs, ) -> None: """ Create a railroad diagram for the parser. Parameters: - - output_html (str or file-like object) - output target for generated + + - ``output_html`` (str or file-like object) - output target for generated diagram HTML - - vertical (int) - threshold for formatting multiple alternatives vertically + - ``vertical`` (int) - threshold for formatting multiple alternatives vertically instead of horizontally (default=3) - - show_results_names - bool flag whether diagram should show annotations for + - ``show_results_names`` - bool flag whether diagram should show annotations for defined results names - - show_groups - bool flag whether groups should be highlighted with an unlabeled surrounding box + - ``show_groups`` - bool flag whether groups should be highlighted with an unlabeled surrounding box + - ``embed`` - bool flag whether generated HTML should omit , , and tags to embed + the resulting HTML in an enclosing HTML source + - ``head`` - str containing additional HTML to insert into the section of the generated code; + can be used to insert custom CSS styling + - ``body`` - str containing additional HTML to insert at the beginning of the section of the + generated code + Additional diagram-formatting keyword arguments can also be included; see railroad.Diagram class. """ @@ -2183,38 +2239,93 @@ class ParserElement(ABC): ) if isinstance(output_html, (str, Path)): with open(output_html, "w", encoding="utf-8") as diag_file: - diag_file.write(railroad_to_html(railroad)) + diag_file.write(railroad_to_html(railroad, embed=embed, **kwargs)) else: # we were passed a file-like object, just write to it - output_html.write(railroad_to_html(railroad)) + output_html.write(railroad_to_html(railroad, embed=embed, **kwargs)) + + # Compatibility synonyms + # fmt: off + @staticmethod + @replaced_by_pep8(inline_literals_using) + def inlineLiteralsUsing(): ... + + @staticmethod + @replaced_by_pep8(set_default_whitespace_chars) + def setDefaultWhitespaceChars(): ... + + @replaced_by_pep8(set_results_name) + def setResultsName(self): ... + + @replaced_by_pep8(set_break) + def setBreak(self): ... + + @replaced_by_pep8(set_parse_action) + def setParseAction(self): ... + + @replaced_by_pep8(add_parse_action) + def addParseAction(self): ... + + @replaced_by_pep8(add_condition) + def addCondition(self): ... + + @replaced_by_pep8(set_fail_action) + def setFailAction(self): ... + + @replaced_by_pep8(try_parse) + def tryParse(self): ... + + @staticmethod + @replaced_by_pep8(enable_left_recursion) + def enableLeftRecursion(): ... + + @staticmethod + @replaced_by_pep8(enable_packrat) + def enablePackrat(): ... + + @replaced_by_pep8(parse_string) + def parseString(self): ... + + @replaced_by_pep8(scan_string) + def scanString(self): ... + + @replaced_by_pep8(transform_string) + def transformString(self): ... + + @replaced_by_pep8(search_string) + def searchString(self): ... + + @replaced_by_pep8(ignore_whitespace) + def ignoreWhitespace(self): ... + + @replaced_by_pep8(leave_whitespace) + def leaveWhitespace(self): ... + + @replaced_by_pep8(set_whitespace_chars) + def setWhitespaceChars(self): ... + + @replaced_by_pep8(parse_with_tabs) + def parseWithTabs(self): ... + + @replaced_by_pep8(set_debug_actions) + def setDebugActions(self): ... + + @replaced_by_pep8(set_debug) + def setDebug(self): ... + + @replaced_by_pep8(set_name) + def setName(self): ... + + @replaced_by_pep8(parse_file) + def parseFile(self): ... + + @replaced_by_pep8(run_tests) + def runTests(self): ... - setDefaultWhitespaceChars = set_default_whitespace_chars - inlineLiteralsUsing = inline_literals_using - setResultsName = set_results_name - setBreak = set_break - setParseAction = set_parse_action - addParseAction = add_parse_action - addCondition = add_condition - setFailAction = set_fail_action - tryParse = try_parse canParseNext = can_parse_next resetCache = reset_cache - enableLeftRecursion = enable_left_recursion - enablePackrat = enable_packrat - parseString = parse_string - scanString = scan_string - searchString = search_string - transformString = transform_string - setWhitespaceChars = set_whitespace_chars - parseWithTabs = parse_with_tabs - setDebugActions = set_debug_actions - setDebug = set_debug defaultName = default_name - setName = set_name - parseFile = parse_file - runTests = run_tests - ignoreWhitespace = ignore_whitespace - leaveWhitespace = leave_whitespace + # fmt: on class _PendingSkip(ParserElement): @@ -2225,7 +2336,7 @@ class _PendingSkip(ParserElement): self.anchor = expr self.must_skip = must_skip - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return str(self.anchor + Empty()).replace("Empty", "...") def __add__(self, other) -> "ParserElement": @@ -2266,21 +2377,10 @@ class Token(ParserElement): def __init__(self): super().__init__(savelist=False) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return type(self).__name__ -class Empty(Token): - """ - An empty token, will always match. - """ - - def __init__(self): - super().__init__() - self.mayReturnEmpty = True - self.mayIndexError = False - - class NoMatch(Token): """ A token that will never match. @@ -2312,25 +2412,33 @@ class Literal(Token): use :class:`Keyword` or :class:`CaselessKeyword`. """ + def __new__(cls, match_string: str = "", *, matchString: str = ""): + # Performance tuning: select a subclass with optimized parseImpl + if cls is Literal: + match_string = matchString or match_string + if not match_string: + return super().__new__(Empty) + if len(match_string) == 1: + return super().__new__(_SingleCharLiteral) + + # Default behavior + return super().__new__(cls) + + # Needed to make copy.copy() work correctly if we customize __new__ + def __getnewargs__(self): + return (self.match,) + def __init__(self, match_string: str = "", *, matchString: str = ""): super().__init__() match_string = matchString or match_string self.match = match_string self.matchLen = len(match_string) - try: - self.firstMatchChar = match_string[0] - except IndexError: - raise ValueError("null string passed to Literal; use Empty() instead") + self.firstMatchChar = match_string[:1] self.errmsg = "Expected " + self.name self.mayReturnEmpty = False self.mayIndexError = False - # Performance tuning: modify __class__ to select - # a parseImpl optimized for single-character check - if self.matchLen == 1 and type(self) is Literal: - self.__class__ = _SingleCharLiteral - - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return repr(self.match) def parseImpl(self, instring, loc, doActions=True): @@ -2341,6 +2449,23 @@ class Literal(Token): raise ParseException(instring, loc, self.errmsg, self) +class Empty(Literal): + """ + An empty token, will always match. + """ + + def __init__(self, match_string="", *, matchString=""): + super().__init__("") + self.mayReturnEmpty = True + self.mayIndexError = False + + def _generateDefaultName(self) -> str: + return "Empty" + + def parseImpl(self, instring, loc, doActions=True): + return loc, [] + + class _SingleCharLiteral(Literal): def parseImpl(self, instring, loc, doActions=True): if instring[loc] == self.firstMatchChar: @@ -2354,8 +2479,8 @@ ParserElement._literalStringClass = Literal class Keyword(Token): """ Token to exactly match a specified string as a keyword, that is, - it must be immediately followed by a non-keyword character. Compare - with :class:`Literal`: + it must be immediately preceded and followed by whitespace or + non-keyword characters. Compare with :class:`Literal`: - ``Literal("if")`` will match the leading ``'if'`` in ``'ifAndOnlyIf'``. @@ -2365,7 +2490,7 @@ class Keyword(Token): Accepts two optional constructor arguments in addition to the keyword string: - - ``identChars`` is a string of characters that would be valid + - ``ident_chars`` is a string of characters that would be valid identifier characters, defaulting to all alphanumerics + "_" and "$" - ``caseless`` allows case-insensitive matching, default is ``False``. @@ -2400,7 +2525,7 @@ class Keyword(Token): self.firstMatchChar = match_string[0] except IndexError: raise ValueError("null string passed to Keyword; use Empty() instead") - self.errmsg = "Expected {} {}".format(type(self).__name__, self.name) + self.errmsg = f"Expected {type(self).__name__} {self.name}" self.mayReturnEmpty = False self.mayIndexError = False self.caseless = caseless @@ -2409,7 +2534,7 @@ class Keyword(Token): identChars = identChars.upper() self.identChars = set(identChars) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return repr(self.match) def parseImpl(self, instring, loc, doActions=True): @@ -2559,7 +2684,7 @@ class CloseMatch(Token): def __init__( self, match_string: str, - max_mismatches: int = None, + max_mismatches: typing.Optional[int] = None, *, maxMismatches: int = 1, caseless=False, @@ -2568,15 +2693,13 @@ class CloseMatch(Token): super().__init__() self.match_string = match_string self.maxMismatches = maxMismatches - self.errmsg = "Expected {!r} (with up to {} mismatches)".format( - self.match_string, self.maxMismatches - ) + self.errmsg = f"Expected {self.match_string!r} (with up to {self.maxMismatches} mismatches)" self.caseless = caseless self.mayIndexError = False self.mayReturnEmpty = False - def _generateDefaultName(self): - return "{}:{!r}".format(type(self).__name__, self.match_string) + def _generateDefaultName(self) -> str: + return f"{type(self).__name__}:{self.match_string!r}" def parseImpl(self, instring, loc, doActions=True): start = loc @@ -2612,7 +2735,9 @@ class CloseMatch(Token): class Word(Token): """Token for matching words composed of allowed character sets. + Parameters: + - ``init_chars`` - string of all characters that should be used to match as a word; "ABC" will match "AAA", "ABAB", "CBAC", etc.; if ``body_chars`` is also specified, then this is the string of @@ -2697,26 +2822,24 @@ class Word(Token): super().__init__() if not initChars: raise ValueError( - "invalid {}, initChars cannot be empty string".format( - type(self).__name__ - ) + f"invalid {type(self).__name__}, initChars cannot be empty string" ) - initChars = set(initChars) - self.initChars = initChars + initChars_set = set(initChars) if excludeChars: - excludeChars = set(excludeChars) - initChars -= excludeChars + excludeChars_set = set(excludeChars) + initChars_set -= excludeChars_set if bodyChars: - bodyChars = set(bodyChars) - excludeChars - self.initCharsOrig = "".join(sorted(initChars)) + bodyChars = "".join(set(bodyChars) - excludeChars_set) + self.initChars = initChars_set + self.initCharsOrig = "".join(sorted(initChars_set)) if bodyChars: - self.bodyCharsOrig = "".join(sorted(bodyChars)) self.bodyChars = set(bodyChars) + self.bodyCharsOrig = "".join(sorted(bodyChars)) else: - self.bodyCharsOrig = "".join(sorted(initChars)) - self.bodyChars = set(initChars) + self.bodyChars = initChars_set + self.bodyCharsOrig = self.initCharsOrig self.maxSpecified = max > 0 @@ -2725,6 +2848,11 @@ class Word(Token): "cannot specify a minimum length < 1; use Opt(Word()) if zero-length word is permitted" ) + if self.maxSpecified and min > max: + raise ValueError( + f"invalid args, if min and max both specified min must be <= max (min={min}, max={max})" + ) + self.minLen = min if max > 0: @@ -2733,62 +2861,66 @@ class Word(Token): self.maxLen = _MAX_INT if exact > 0: + min = max = exact self.maxLen = exact self.minLen = exact self.errmsg = "Expected " + self.name self.mayIndexError = False self.asKeyword = asKeyword + if self.asKeyword: + self.errmsg += " as a keyword" # see if we can make a regex for this Word - if " " not in self.initChars | self.bodyChars and (min == 1 and exact == 0): + if " " not in (self.initChars | self.bodyChars): + if len(self.initChars) == 1: + re_leading_fragment = re.escape(self.initCharsOrig) + else: + re_leading_fragment = f"[{_collapse_string_to_ranges(self.initChars)}]" + if self.bodyChars == self.initChars: if max == 0: repeat = "+" elif max == 1: repeat = "" else: - repeat = "{{{},{}}}".format( - self.minLen, "" if self.maxLen == _MAX_INT else self.maxLen - ) - self.reString = "[{}]{}".format( - _collapse_string_to_ranges(self.initChars), - repeat, - ) - elif len(self.initChars) == 1: - if max == 0: - repeat = "*" - else: - repeat = "{{0,{}}}".format(max - 1) - self.reString = "{}[{}]{}".format( - re.escape(self.initCharsOrig), - _collapse_string_to_ranges(self.bodyChars), - repeat, - ) + if self.minLen != self.maxLen: + repeat = f"{{{self.minLen},{'' if self.maxLen == _MAX_INT else self.maxLen}}}" + else: + repeat = f"{{{self.minLen}}}" + self.reString = f"{re_leading_fragment}{repeat}" else: - if max == 0: - repeat = "*" - elif max == 2: + if max == 1: + re_body_fragment = "" repeat = "" else: - repeat = "{{0,{}}}".format(max - 1) - self.reString = "[{}][{}]{}".format( - _collapse_string_to_ranges(self.initChars), - _collapse_string_to_ranges(self.bodyChars), - repeat, + re_body_fragment = f"[{_collapse_string_to_ranges(self.bodyChars)}]" + if max == 0: + repeat = "*" + elif max == 2: + repeat = "?" if min <= 1 else "" + else: + if min != max: + repeat = f"{{{min - 1 if min > 0 else 0},{max - 1}}}" + else: + repeat = f"{{{min - 1 if min > 0 else 0}}}" + + self.reString = ( + f"{re_leading_fragment}" f"{re_body_fragment}" f"{repeat}" ) + if self.asKeyword: - self.reString = r"\b" + self.reString + r"\b" + self.reString = rf"\b{self.reString}\b" try: self.re = re.compile(self.reString) except re.error: - self.re = None + self.re = None # type: ignore[assignment] else: self.re_match = self.re.match - self.__class__ = _WordRegex + self.parseImpl = self.parseImpl_regex # type: ignore[assignment] - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: def charsAsStr(s): max_repr_len = 16 s = _collapse_string_to_ranges(s, re_escape=False) @@ -2798,11 +2930,9 @@ class Word(Token): return s if self.initChars != self.bodyChars: - base = "W:({}, {})".format( - charsAsStr(self.initChars), charsAsStr(self.bodyChars) - ) + base = f"W:({charsAsStr(self.initChars)}, {charsAsStr(self.bodyChars)})" else: - base = "W:({})".format(charsAsStr(self.initChars)) + base = f"W:({charsAsStr(self.initChars)})" # add length specification if self.minLen > 1 or self.maxLen != _MAX_INT: @@ -2810,11 +2940,11 @@ class Word(Token): if self.minLen == 1: return base[2:] else: - return base + "{{{}}}".format(self.minLen) + return base + f"{{{self.minLen}}}" elif self.maxLen == _MAX_INT: - return base + "{{{},...}}".format(self.minLen) + return base + f"{{{self.minLen},...}}" else: - return base + "{{{},{}}}".format(self.minLen, self.maxLen) + return base + f"{{{self.minLen},{self.maxLen}}}" return base def parseImpl(self, instring, loc, doActions=True): @@ -2849,9 +2979,7 @@ class Word(Token): return loc, instring[start:loc] - -class _WordRegex(Word): - def parseImpl(self, instring, loc, doActions=True): + def parseImpl_regex(self, instring, loc, doActions=True): result = self.re_match(instring, loc) if not result: raise ParseException(instring, loc, self.errmsg, self) @@ -2860,7 +2988,7 @@ class _WordRegex(Word): return loc, result.group() -class Char(_WordRegex): +class Char(Word): """A short-cut class for defining :class:`Word` ``(characters, exact=1)``, when defining a match of any single character in a string of characters. @@ -2878,13 +3006,8 @@ class Char(_WordRegex): asKeyword = asKeyword or as_keyword excludeChars = excludeChars or exclude_chars super().__init__( - charset, exact=1, asKeyword=asKeyword, excludeChars=excludeChars + charset, exact=1, as_keyword=asKeyword, exclude_chars=excludeChars ) - self.reString = "[{}]".format(_collapse_string_to_ranges(self.initChars)) - if asKeyword: - self.reString = r"\b{}\b".format(self.reString) - self.re = re.compile(self.reString) - self.re_match = self.re.match class Regex(Token): @@ -2954,9 +3077,9 @@ class Regex(Token): self.asGroupList = asGroupList self.asMatch = asMatch if self.asGroupList: - self.parseImpl = self.parseImplAsGroupList + self.parseImpl = self.parseImplAsGroupList # type: ignore [assignment] if self.asMatch: - self.parseImpl = self.parseImplAsMatch + self.parseImpl = self.parseImplAsMatch # type: ignore [assignment] @cached_property def re(self): @@ -2966,9 +3089,7 @@ class Regex(Token): try: return re.compile(self.pattern, self.flags) except re.error: - raise ValueError( - "invalid pattern ({!r}) passed to Regex".format(self.pattern) - ) + raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex") @cached_property def re_match(self): @@ -2978,7 +3099,7 @@ class Regex(Token): def mayReturnEmpty(self): return self.re_match("") is not None - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "Re:({})".format(repr(self.pattern).replace("\\\\", "\\")) def parseImpl(self, instring, loc, doActions=True): @@ -3024,10 +3145,12 @@ class Regex(Token): # prints "

main title

" """ if self.asGroupList: - raise TypeError("cannot use sub() with Regex(asGroupList=True)") + raise TypeError("cannot use sub() with Regex(as_group_list=True)") if self.asMatch and callable(repl): - raise TypeError("cannot use sub() with a callable with Regex(asMatch=True)") + raise TypeError( + "cannot use sub() with a callable with Regex(as_match=True)" + ) if self.asMatch: @@ -3081,7 +3204,7 @@ class QuotedString(Token): [['This is the "quote"']] [['This is the quote with "embedded" quotes']] """ - ws_map = ((r"\t", "\t"), (r"\n", "\n"), (r"\f", "\f"), (r"\r", "\r")) + ws_map = dict(((r"\t", "\t"), (r"\n", "\n"), (r"\f", "\f"), (r"\r", "\r"))) def __init__( self, @@ -3120,57 +3243,54 @@ class QuotedString(Token): else: endQuoteChar = endQuoteChar.strip() if not endQuoteChar: - raise ValueError("endQuoteChar cannot be the empty string") + raise ValueError("end_quote_char cannot be the empty string") - self.quoteChar = quote_char - self.quoteCharLen = len(quote_char) - self.firstQuoteChar = quote_char[0] - self.endQuoteChar = endQuoteChar - self.endQuoteCharLen = len(endQuoteChar) - self.escChar = escChar - self.escQuote = escQuote - self.unquoteResults = unquoteResults - self.convertWhitespaceEscapes = convertWhitespaceEscapes + self.quoteChar: str = quote_char + self.quoteCharLen: int = len(quote_char) + self.firstQuoteChar: str = quote_char[0] + self.endQuoteChar: str = endQuoteChar + self.endQuoteCharLen: int = len(endQuoteChar) + self.escChar: str = escChar or "" + self.escQuote: str = escQuote or "" + self.unquoteResults: bool = unquoteResults + self.convertWhitespaceEscapes: bool = convertWhitespaceEscapes + self.multiline = multiline sep = "" inner_pattern = "" if escQuote: - inner_pattern += r"{}(?:{})".format(sep, re.escape(escQuote)) + inner_pattern += rf"{sep}(?:{re.escape(escQuote)})" sep = "|" if escChar: - inner_pattern += r"{}(?:{}.)".format(sep, re.escape(escChar)) + inner_pattern += rf"{sep}(?:{re.escape(escChar)}.)" sep = "|" - self.escCharReplacePattern = re.escape(self.escChar) + "(.)" + self.escCharReplacePattern = re.escape(escChar) + "(.)" if len(self.endQuoteChar) > 1: inner_pattern += ( - "{}(?:".format(sep) + f"{sep}(?:" + "|".join( - "(?:{}(?!{}))".format( - re.escape(self.endQuoteChar[:i]), - re.escape(self.endQuoteChar[i:]), - ) + f"(?:{re.escape(self.endQuoteChar[:i])}(?!{re.escape(self.endQuoteChar[i:])}))" for i in range(len(self.endQuoteChar) - 1, 0, -1) ) + ")" ) sep = "|" + self.flags = re.RegexFlag(0) + if multiline: self.flags = re.MULTILINE | re.DOTALL - inner_pattern += r"{}(?:[^{}{}])".format( - sep, - _escape_regex_range_chars(self.endQuoteChar[0]), - (_escape_regex_range_chars(escChar) if escChar is not None else ""), + inner_pattern += ( + rf"{sep}(?:[^{_escape_regex_range_chars(self.endQuoteChar[0])}" + rf"{(_escape_regex_range_chars(escChar) if escChar is not None else '')}])" ) else: - self.flags = 0 - inner_pattern += r"{}(?:[^{}\n\r{}])".format( - sep, - _escape_regex_range_chars(self.endQuoteChar[0]), - (_escape_regex_range_chars(escChar) if escChar is not None else ""), + inner_pattern += ( + rf"{sep}(?:[^{_escape_regex_range_chars(self.endQuoteChar[0])}\n\r" + rf"{(_escape_regex_range_chars(escChar) if escChar is not None else '')}])" ) self.pattern = "".join( @@ -3183,26 +3303,33 @@ class QuotedString(Token): ] ) + if self.unquoteResults: + if self.convertWhitespaceEscapes: + self.unquote_scan_re = re.compile( + rf"({'|'.join(re.escape(k) for k in self.ws_map)})|({re.escape(self.escChar)}.)|(\n|.)", + flags=self.flags, + ) + else: + self.unquote_scan_re = re.compile( + rf"({re.escape(self.escChar)}.)|(\n|.)", flags=self.flags + ) + try: self.re = re.compile(self.pattern, self.flags) self.reString = self.pattern self.re_match = self.re.match except re.error: - raise ValueError( - "invalid pattern {!r} passed to Regex".format(self.pattern) - ) + raise ValueError(f"invalid pattern {self.pattern!r} passed to Regex") self.errmsg = "Expected " + self.name self.mayIndexError = False self.mayReturnEmpty = True - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: if self.quoteChar == self.endQuoteChar and isinstance(self.quoteChar, str_type): - return "string enclosed in {!r}".format(self.quoteChar) + return f"string enclosed in {self.quoteChar!r}" - return "quoted string, starting with {} ending with {}".format( - self.quoteChar, self.endQuoteChar - ) + return f"quoted string, starting with {self.quoteChar} ending with {self.endQuoteChar}" def parseImpl(self, instring, loc, doActions=True): result = ( @@ -3217,19 +3344,24 @@ class QuotedString(Token): ret = result.group() if self.unquoteResults: - # strip off quotes ret = ret[self.quoteCharLen : -self.endQuoteCharLen] if isinstance(ret, str_type): - # replace escaped whitespace - if "\\" in ret and self.convertWhitespaceEscapes: - for wslit, wschar in self.ws_map: - ret = ret.replace(wslit, wschar) - - # replace escaped characters - if self.escChar: - ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret) + if self.convertWhitespaceEscapes: + ret = "".join( + self.ws_map[match.group(1)] + if match.group(1) + else match.group(2)[-1] + if match.group(2) + else match.group(3) + for match in self.unquote_scan_re.finditer(ret) + ) + else: + ret = "".join( + match.group(1)[-1] if match.group(1) else match.group(2) + for match in self.unquote_scan_re.finditer(ret) + ) # replace escaped quotes if self.escQuote: @@ -3252,7 +3384,7 @@ class CharsNotIn(Token): # define a comma-separated-value as anything that is not a ',' csv_value = CharsNotIn(',') - print(delimited_list(csv_value).parse_string("dkls,lsdkjf,s12 34,@!#,213")) + print(DelimitedList(csv_value).parse_string("dkls,lsdkjf,s12 34,@!#,213")) prints:: @@ -3294,12 +3426,12 @@ class CharsNotIn(Token): self.mayReturnEmpty = self.minLen == 0 self.mayIndexError = False - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: not_chars_str = _collapse_string_to_ranges(self.notChars) if len(not_chars_str) > 16: - return "!W:({}...)".format(self.notChars[: 16 - 3]) + return f"!W:({self.notChars[: 16 - 3]}...)" else: - return "!W:({})".format(self.notChars) + return f"!W:({self.notChars})" def parseImpl(self, instring, loc, doActions=True): notchars = self.notCharsSet @@ -3376,7 +3508,7 @@ class White(Token): self.maxLen = exact self.minLen = exact - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "".join(White.whiteStrs[c] for c in self.matchWhite) def parseImpl(self, instring, loc, doActions=True): @@ -3411,7 +3543,7 @@ class GoToColumn(PositionToken): super().__init__() self.col = colno - def preParse(self, instring, loc): + def preParse(self, instring: str, loc: int) -> int: if col(loc, instring) != self.col: instrlen = len(instring) if self.ignoreExprs: @@ -3446,7 +3578,7 @@ class LineStart(PositionToken): B AAA and definitely not this one ''' - for t in (LineStart() + 'AAA' + restOfLine).search_string(test): + for t in (LineStart() + 'AAA' + rest_of_line).search_string(test): print(t) prints:: @@ -3464,7 +3596,7 @@ class LineStart(PositionToken): self.skipper = Empty().set_whitespace_chars(self.whiteChars) self.errmsg = "Expected start of line" - def preParse(self, instring, loc): + def preParse(self, instring: str, loc: int) -> int: if loc == 0: return loc else: @@ -3624,7 +3756,7 @@ class ParseExpression(ParserElement): self.exprs = [exprs] self.callPreparse = False - def recurse(self) -> Sequence[ParserElement]: + def recurse(self) -> List[ParserElement]: return self.exprs[:] def append(self, other) -> ParserElement: @@ -3669,8 +3801,8 @@ class ParseExpression(ParserElement): e.ignore(self.ignoreExprs[-1]) return self - def _generateDefaultName(self): - return "{}:({})".format(self.__class__.__name__, str(self.exprs)) + def _generateDefaultName(self) -> str: + return f"{self.__class__.__name__}:({str(self.exprs)})" def streamline(self) -> ParserElement: if self.streamlined: @@ -3714,6 +3846,11 @@ class ParseExpression(ParserElement): return self def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + DeprecationWarning, + stacklevel=2, + ) tmp = (validateTrace if validateTrace is not None else [])[:] + [self] for e in self.exprs: e.validate(tmp) @@ -3721,6 +3858,7 @@ class ParseExpression(ParserElement): def copy(self) -> ParserElement: ret = super().copy() + ret = typing.cast(ParseExpression, ret) ret.exprs = [e.copy() for e in self.exprs] return ret @@ -3750,8 +3888,14 @@ class ParseExpression(ParserElement): return super()._setResultsName(name, listAllMatches) - ignoreWhitespace = ignore_whitespace - leaveWhitespace = leave_whitespace + # Compatibility synonyms + # fmt: off + @replaced_by_pep8(leave_whitespace) + def leaveWhitespace(self): ... + + @replaced_by_pep8(ignore_whitespace) + def ignoreWhitespace(self): ... + # fmt: on class And(ParseExpression): @@ -3777,7 +3921,7 @@ class And(ParseExpression): super().__init__(*args, **kwargs) self.leave_whitespace() - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "-" def __init__( @@ -3789,7 +3933,9 @@ class And(ParseExpression): for i, expr in enumerate(exprs): if expr is Ellipsis: if i < len(exprs) - 1: - skipto_arg: ParserElement = (Empty() + exprs[i + 1]).exprs[-1] + skipto_arg: ParserElement = typing.cast( + ParseExpression, (Empty() + exprs[i + 1]) + ).exprs[-1] tmp.append(SkipTo(skipto_arg)("_skipped*")) else: raise Exception( @@ -3822,8 +3968,9 @@ class And(ParseExpression): and isinstance(e.exprs[-1], _PendingSkip) for e in self.exprs[:-1] ): + deleted_expr_marker = NoMatch() for i, e in enumerate(self.exprs[:-1]): - if e is None: + if e is deleted_expr_marker: continue if ( isinstance(e, ParseExpression) @@ -3831,17 +3978,19 @@ class And(ParseExpression): and isinstance(e.exprs[-1], _PendingSkip) ): e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1] - self.exprs[i + 1] = None - self.exprs = [e for e in self.exprs if e is not None] + self.exprs[i + 1] = deleted_expr_marker + self.exprs = [e for e in self.exprs if e is not deleted_expr_marker] super().streamline() # link any IndentedBlocks to the prior expression + prev: ParserElement + cur: ParserElement for prev, cur in zip(self.exprs, self.exprs[1:]): # traverse cur or any first embedded expr of cur looking for an IndentedBlock # (but watch out for recursive grammar) seen = set() - while cur: + while True: if id(cur) in seen: break seen.add(id(cur)) @@ -3853,7 +4002,10 @@ class And(ParseExpression): ) break subs = cur.recurse() - cur = next(iter(subs), None) + next_first = next(iter(subs), None) + if next_first is None: + break + cur = typing.cast(ParserElement, next_first) self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) return self @@ -3884,13 +4036,14 @@ class And(ParseExpression): ) else: loc, exprtokens = e._parse(instring, loc, doActions) - if exprtokens or exprtokens.haskeys(): - resultlist += exprtokens + resultlist += exprtokens return loc, resultlist def __iadd__(self, other): if isinstance(other, str_type): other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented return self.append(other) # And([self, other]) def _checkRecursion(self, parseElementList): @@ -3900,7 +4053,7 @@ class And(ParseExpression): if not e.mayReturnEmpty: break - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: inner = " ".join(str(e) for e in self.exprs) # strip off redundant inner {}'s while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}": @@ -3958,7 +4111,7 @@ class Or(ParseExpression): loc2 = e.try_parse(instring, loc, raise_fatal=True) except ParseFatalException as pfe: pfe.__traceback__ = None - pfe.parserElement = e + pfe.parser_element = e fatals.append(pfe) maxException = None maxExcLoc = -1 @@ -4016,12 +4169,15 @@ class Or(ParseExpression): if len(fatals) > 1: fatals.sort(key=lambda e: -e.loc) if fatals[0].loc == fatals[1].loc: - fatals.sort(key=lambda e: (-e.loc, -len(str(e.parserElement)))) + fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element)))) max_fatal = fatals[0] raise max_fatal if maxException is not None: - maxException.msg = self.errmsg + # infer from this check that all alternatives failed at the current position + # so emit this collective error message instead of any single error message + if maxExcLoc == loc: + maxException.msg = self.errmsg raise maxException else: raise ParseException( @@ -4031,9 +4187,11 @@ class Or(ParseExpression): def __ixor__(self, other): if isinstance(other, str_type): other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented return self.append(other) # Or([self, other]) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "{" + " ^ ".join(str(e) for e in self.exprs) + "}" def _setResultsName(self, name, listAllMatches=False): @@ -4118,7 +4276,7 @@ class MatchFirst(ParseExpression): ) except ParseFatalException as pfe: pfe.__traceback__ = None - pfe.parserElement = e + pfe.parser_element = e raise except ParseException as err: if err.loc > maxExcLoc: @@ -4132,7 +4290,10 @@ class MatchFirst(ParseExpression): maxExcLoc = len(instring) if maxException is not None: - maxException.msg = self.errmsg + # infer from this check that all alternatives failed at the current position + # so emit this collective error message instead of any individual error message + if maxExcLoc == loc: + maxException.msg = self.errmsg raise maxException else: raise ParseException( @@ -4142,9 +4303,11 @@ class MatchFirst(ParseExpression): def __ior__(self, other): if isinstance(other, str_type): other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented return self.append(other) # MatchFirst([self, other]) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "{" + " | ".join(str(e) for e in self.exprs) + "}" def _setResultsName(self, name, listAllMatches=False): @@ -4242,6 +4405,13 @@ class Each(ParseExpression): self.initExprGroups = True self.saveAsList = True + def __iand__(self, other): + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return self.append(other) # Each([self, other]) + def streamline(self) -> ParserElement: super().streamline() if self.exprs: @@ -4296,7 +4466,7 @@ class Each(ParseExpression): tmpLoc = e.try_parse(instring, tmpLoc, raise_fatal=True) except ParseFatalException as pfe: pfe.__traceback__ = None - pfe.parserElement = e + pfe.parser_element = e fatals.append(pfe) failed.append(e) except ParseException: @@ -4315,7 +4485,7 @@ class Each(ParseExpression): if len(fatals) > 1: fatals.sort(key=lambda e: -e.loc) if fatals[0].loc == fatals[1].loc: - fatals.sort(key=lambda e: (-e.loc, -len(str(e.parserElement)))) + fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element)))) max_fatal = fatals[0] raise max_fatal @@ -4324,7 +4494,7 @@ class Each(ParseExpression): raise ParseException( instring, loc, - "Missing one or more required elements ({})".format(missing), + f"Missing one or more required elements ({missing})", ) # add any unmatched Opts, in case they have default values defined @@ -4337,7 +4507,7 @@ class Each(ParseExpression): return loc, total_results - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "{" + " & ".join(str(e) for e in self.exprs) + "}" @@ -4349,12 +4519,14 @@ class ParseElementEnhance(ParserElement): def __init__(self, expr: Union[ParserElement, str], savelist: bool = False): super().__init__(savelist) if isinstance(expr, str_type): + expr_str = typing.cast(str, expr) if issubclass(self._literalStringClass, Token): - expr = self._literalStringClass(expr) + expr = self._literalStringClass(expr_str) # type: ignore[call-arg] elif issubclass(type(self), self._literalStringClass): - expr = Literal(expr) + expr = Literal(expr_str) else: - expr = self._literalStringClass(Literal(expr)) + expr = self._literalStringClass(Literal(expr_str)) # type: ignore[assignment, call-arg] + expr = typing.cast(ParserElement, expr) self.expr = expr if expr is not None: self.mayIndexError = expr.mayIndexError @@ -4367,12 +4539,16 @@ class ParseElementEnhance(ParserElement): self.callPreparse = expr.callPreparse self.ignoreExprs.extend(expr.ignoreExprs) - def recurse(self) -> Sequence[ParserElement]: + def recurse(self) -> List[ParserElement]: return [self.expr] if self.expr is not None else [] def parseImpl(self, instring, loc, doActions=True): if self.expr is not None: - return self.expr._parse(instring, loc, doActions, callPreParse=False) + try: + return self.expr._parse(instring, loc, doActions, callPreParse=False) + except ParseBaseException as pbe: + pbe.msg = self.errmsg + raise else: raise ParseException(instring, loc, "No expression defined", self) @@ -4380,8 +4556,8 @@ class ParseElementEnhance(ParserElement): super().leave_whitespace(recursive) if recursive: - self.expr = self.expr.copy() if self.expr is not None: + self.expr = self.expr.copy() self.expr.leave_whitespace(recursive) return self @@ -4389,8 +4565,8 @@ class ParseElementEnhance(ParserElement): super().ignore_whitespace(recursive) if recursive: - self.expr = self.expr.copy() if self.expr is not None: + self.expr = self.expr.copy() self.expr.ignore_whitespace(recursive) return self @@ -4420,6 +4596,11 @@ class ParseElementEnhance(ParserElement): self.expr._checkRecursion(subRecCheckList) def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + DeprecationWarning, + stacklevel=2, + ) if validateTrace is None: validateTrace = [] tmp = validateTrace[:] + [self] @@ -4427,11 +4608,17 @@ class ParseElementEnhance(ParserElement): self.expr.validate(tmp) self._checkRecursion([]) - def _generateDefaultName(self): - return "{}:({})".format(self.__class__.__name__, str(self.expr)) + def _generateDefaultName(self) -> str: + return f"{self.__class__.__name__}:({str(self.expr)})" - ignoreWhitespace = ignore_whitespace - leaveWhitespace = leave_whitespace + # Compatibility synonyms + # fmt: off + @replaced_by_pep8(leave_whitespace) + def leaveWhitespace(self): ... + + @replaced_by_pep8(ignore_whitespace) + def ignoreWhitespace(self): ... + # fmt: on class IndentedBlock(ParseElementEnhance): @@ -4443,13 +4630,13 @@ class IndentedBlock(ParseElementEnhance): class _Indent(Empty): def __init__(self, ref_col: int): super().__init__() - self.errmsg = "expected indent at column {}".format(ref_col) + self.errmsg = f"expected indent at column {ref_col}" self.add_condition(lambda s, l, t: col(l, s) == ref_col) class _IndentGreater(Empty): def __init__(self, ref_col: int): super().__init__() - self.errmsg = "expected indent at column greater than {}".format(ref_col) + self.errmsg = f"expected indent at column greater than {ref_col}" self.add_condition(lambda s, l, t: col(l, s) > ref_col) def __init__( @@ -4469,7 +4656,7 @@ class IndentedBlock(ParseElementEnhance): # see if self.expr matches at the current location - if not it will raise an exception # and no further work is necessary - self.expr.try_parse(instring, anchor_loc, doActions) + self.expr.try_parse(instring, anchor_loc, do_actions=doActions) indent_col = col(anchor_loc, instring) peer_detect_expr = self._Indent(indent_col) @@ -4532,7 +4719,7 @@ class AtLineStart(ParseElementEnhance): B AAA and definitely not this one ''' - for t in (AtLineStart('AAA') + restOfLine).search_string(test): + for t in (AtLineStart('AAA') + rest_of_line).search_string(test): print(t) prints:: @@ -4598,9 +4785,9 @@ class PrecededBy(ParseElementEnhance): Parameters: - - expr - expression that must match prior to the current parse + - ``expr`` - expression that must match prior to the current parse location - - retreat - (default= ``None``) - (int) maximum number of characters + - ``retreat`` - (default= ``None``) - (int) maximum number of characters to lookbehind prior to the current parse location If the lookbehind expression is a string, :class:`Literal`, @@ -4627,6 +4814,7 @@ class PrecededBy(ParseElementEnhance): self.mayIndexError = False self.exact = False if isinstance(expr, str_type): + expr = typing.cast(str, expr) retreat = len(expr) self.exact = True elif isinstance(expr, (Literal, Keyword)): @@ -4746,18 +4934,18 @@ class NotAny(ParseElementEnhance): self.errmsg = "Found unwanted token, " + str(self.expr) def parseImpl(self, instring, loc, doActions=True): - if self.expr.can_parse_next(instring, loc): + if self.expr.can_parse_next(instring, loc, do_actions=doActions): raise ParseException(instring, loc, self.errmsg, self) return loc, [] - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "~{" + str(self.expr) + "}" class _MultipleMatch(ParseElementEnhance): def __init__( self, - expr: ParserElement, + expr: Union[str, ParserElement], stop_on: typing.Optional[Union[ParserElement, str]] = None, *, stopOn: typing.Optional[Union[ParserElement, str]] = None, @@ -4781,7 +4969,7 @@ class _MultipleMatch(ParseElementEnhance): self_skip_ignorables = self._skipIgnorables check_ender = self.not_ender is not None if check_ender: - try_not_ender = self.not_ender.tryParse + try_not_ender = self.not_ender.try_parse # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) @@ -4798,8 +4986,7 @@ class _MultipleMatch(ParseElementEnhance): else: preloc = loc loc, tmptokens = self_expr_parse(instring, preloc, doActions) - if tmptokens or tmptokens.haskeys(): - tokens += tmptokens + tokens += tmptokens except (ParseException, IndexError): pass @@ -4837,10 +5024,11 @@ class OneOrMore(_MultipleMatch): Repetition of one or more of the given expression. Parameters: - - expr - expression that must match one or more times - - stop_on - (default= ``None``) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + + - ``expr`` - expression that must match one or more times + - ``stop_on`` - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) Example:: @@ -4859,7 +5047,7 @@ class OneOrMore(_MultipleMatch): (attr_expr * (1,)).parse_string(text).pprint() """ - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "{" + str(self.expr) + "}..." @@ -4868,6 +5056,7 @@ class ZeroOrMore(_MultipleMatch): Optional repetition of zero or more of the given expression. Parameters: + - ``expr`` - expression that must match zero or more times - ``stop_on`` - expression for a terminating sentinel (only required if the sentinel would ordinarily match the repetition @@ -4878,7 +5067,7 @@ class ZeroOrMore(_MultipleMatch): def __init__( self, - expr: ParserElement, + expr: Union[str, ParserElement], stop_on: typing.Optional[Union[ParserElement, str]] = None, *, stopOn: typing.Optional[Union[ParserElement, str]] = None, @@ -4892,10 +5081,75 @@ class ZeroOrMore(_MultipleMatch): except (ParseException, IndexError): return loc, ParseResults([], name=self.resultsName) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: return "[" + str(self.expr) + "]..." +class DelimitedList(ParseElementEnhance): + def __init__( + self, + expr: Union[str, ParserElement], + delim: Union[str, ParserElement] = ",", + combine: bool = False, + min: typing.Optional[int] = None, + max: typing.Optional[int] = None, + *, + allow_trailing_delim: bool = False, + ): + """Helper to define a delimited list of expressions - the delimiter + defaults to ','. By default, the list elements and delimiters can + have intervening whitespace, and comments, but this can be + overridden by passing ``combine=True`` in the constructor. If + ``combine`` is set to ``True``, the matching tokens are + returned as a single token string, with the delimiters included; + otherwise, the matching tokens are returned as a list of tokens, + with the delimiters suppressed. + + If ``allow_trailing_delim`` is set to True, then the list may end with + a delimiter. + + Example:: + + DelimitedList(Word(alphas)).parse_string("aa,bb,cc") # -> ['aa', 'bb', 'cc'] + DelimitedList(Word(hexnums), delim=':', combine=True).parse_string("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] + """ + if isinstance(expr, str_type): + expr = ParserElement._literalStringClass(expr) + expr = typing.cast(ParserElement, expr) + + if min is not None: + if min < 1: + raise ValueError("min must be greater than 0") + if max is not None: + if min is not None and max < min: + raise ValueError("max must be greater than, or equal to min") + + self.content = expr + self.raw_delim = str(delim) + self.delim = delim + self.combine = combine + if not combine: + self.delim = Suppress(delim) + self.min = min or 1 + self.max = max + self.allow_trailing_delim = allow_trailing_delim + + delim_list_expr = self.content + (self.delim + self.content) * ( + self.min - 1, + None if self.max is None else self.max - 1, + ) + if self.allow_trailing_delim: + delim_list_expr += Opt(self.delim) + + if self.combine: + delim_list_expr = Combine(delim_list_expr) + + super().__init__(delim_list_expr, savelist=True) + + def _generateDefaultName(self) -> str: + return "{0} [{1} {0}]...".format(self.content.streamline(), self.raw_delim) + + class _NullToken: def __bool__(self): return False @@ -4909,6 +5163,7 @@ class Opt(ParseElementEnhance): Optional matching of the given expression. Parameters: + - ``expr`` - expression that must match zero or more times - ``default`` (optional) - value to be returned if the optional expression is not found. @@ -4969,7 +5224,7 @@ class Opt(ParseElementEnhance): tokens = [] return loc, tokens - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: inner = str(self.expr) # strip off redundant inner {}'s while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}": @@ -4986,6 +5241,7 @@ class SkipTo(ParseElementEnhance): expression is found. Parameters: + - ``expr`` - target expression marking the end of the data to be skipped - ``include`` - if ``True``, the target expression is also parsed (the skipped text and target expression are returned as a 2-element @@ -5045,14 +5301,15 @@ class SkipTo(ParseElementEnhance): self, other: Union[ParserElement, str], include: bool = False, - ignore: bool = None, + ignore: typing.Optional[Union[ParserElement, str]] = None, fail_on: typing.Optional[Union[ParserElement, str]] = None, *, - failOn: Union[ParserElement, str] = None, + failOn: typing.Optional[Union[ParserElement, str]] = None, ): super().__init__(other) failOn = failOn or fail_on - self.ignoreExpr = ignore + if ignore is not None: + self.ignore(ignore) self.mayReturnEmpty = True self.mayIndexError = False self.includeMatch = include @@ -5070,9 +5327,7 @@ class SkipTo(ParseElementEnhance): self_failOn_canParseNext = ( self.failOn.canParseNext if self.failOn is not None else None ) - self_ignoreExpr_tryParse = ( - self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - ) + self_preParse = self.preParse if self.callPreparse else None tmploc = loc while tmploc <= instrlen: @@ -5081,13 +5336,9 @@ class SkipTo(ParseElementEnhance): if self_failOn_canParseNext(instring, tmploc): break - if self_ignoreExpr_tryParse is not None: - # advance past ignore expressions - while 1: - try: - tmploc = self_ignoreExpr_tryParse(instring, tmploc) - except ParseBaseException: - break + if self_preParse is not None: + # skip grammar-ignored expressions + tmploc = self_preParse(instring, tmploc) try: self_expr_parse(instring, tmploc, doActions=False, callPreParse=False) @@ -5145,15 +5396,20 @@ class Forward(ParseElementEnhance): def __init__(self, other: typing.Optional[Union[ParserElement, str]] = None): self.caller_frame = traceback.extract_stack(limit=2)[0] - super().__init__(other, savelist=False) + super().__init__(other, savelist=False) # type: ignore[arg-type] self.lshift_line = None - def __lshift__(self, other): + def __lshift__(self, other) -> "Forward": if hasattr(self, "caller_frame"): del self.caller_frame if isinstance(other, str_type): other = self._literalStringClass(other) + + if not isinstance(other, ParserElement): + return NotImplemented + self.expr = other + self.streamlined = other.streamlined self.mayIndexError = self.expr.mayIndexError self.mayReturnEmpty = self.expr.mayReturnEmpty self.set_whitespace_chars( @@ -5162,13 +5418,16 @@ class Forward(ParseElementEnhance): self.skipWhitespace = self.expr.skipWhitespace self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) - self.lshift_line = traceback.extract_stack(limit=2)[-2] + self.lshift_line = traceback.extract_stack(limit=2)[-2] # type: ignore[assignment] return self - def __ilshift__(self, other): + def __ilshift__(self, other) -> "Forward": + if not isinstance(other, ParserElement): + return NotImplemented + return self << other - def __or__(self, other): + def __or__(self, other) -> "ParserElement": caller_line = traceback.extract_stack(limit=2)[-2] if ( __diag__.warn_on_match_first_with_lshift_operator @@ -5205,12 +5464,12 @@ class Forward(ParseElementEnhance): not in self.suppress_warnings_ ): # walk stack until parse_string, scan_string, search_string, or transform_string is found - parse_fns = [ + parse_fns = ( "parse_string", "scan_string", "search_string", "transform_string", - ] + ) tb = traceback.extract_stack(limit=200) for i, frm in enumerate(reversed(tb), start=1): if frm.name in parse_fns: @@ -5308,6 +5567,11 @@ class Forward(ParseElementEnhance): return self def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + DeprecationWarning, + stacklevel=2, + ) if validateTrace is None: validateTrace = [] @@ -5317,7 +5581,7 @@ class Forward(ParseElementEnhance): self.expr.validate(tmp) self._checkRecursion([]) - def _generateDefaultName(self): + def _generateDefaultName(self) -> str: # Avoid infinite recursion by setting a temporary _defaultName self._defaultName = ": ..." @@ -5356,8 +5620,14 @@ class Forward(ParseElementEnhance): return super()._setResultsName(name, list_all_matches) - ignoreWhitespace = ignore_whitespace - leaveWhitespace = leave_whitespace + # Compatibility synonyms + # fmt: off + @replaced_by_pep8(leave_whitespace) + def leaveWhitespace(self): ... + + @replaced_by_pep8(ignore_whitespace) + def ignoreWhitespace(self): ... + # fmt: on class TokenConverter(ParseElementEnhance): @@ -5439,11 +5709,11 @@ class Group(TokenConverter): ident = Word(alphas) num = Word(nums) term = ident | num - func = ident + Opt(delimited_list(term)) + func = ident + Opt(DelimitedList(term)) print(func.parse_string("fn a, b, 100")) # -> ['fn', 'a', 'b', '100'] - func = ident + Group(Opt(delimited_list(term))) + func = ident + Group(Opt(DelimitedList(term))) print(func.parse_string("fn a, b, 100")) # -> ['fn', ['a', 'b', '100']] """ @@ -5579,7 +5849,7 @@ class Suppress(TokenConverter): ['a', 'b', 'c', 'd'] ['START', 'relevant text ', 'END'] - (See also :class:`delimited_list`.) + (See also :class:`DelimitedList`.) """ def __init__(self, expr: Union[ParserElement, str], savelist: bool = False): @@ -5638,15 +5908,13 @@ def trace_parse_action(f: ParseAction) -> ParseAction: s, l, t = paArgs[-3:] if len(paArgs) > 3: thisFunc = paArgs[0].__class__.__name__ + "." + thisFunc - sys.stderr.write( - ">>entering {}(line: {!r}, {}, {!r})\n".format(thisFunc, line(l, s), l, t) - ) + sys.stderr.write(f">>entering {thisFunc}(line: {line(l, s)!r}, {l}, {t!r})\n") try: ret = f(*paArgs) except Exception as exc: - sys.stderr.write("< str: ) try: return "".join(_expanded(part) for part in _reBracketExpr.parse_string(s).body) - except Exception: + except Exception as e: return "" @@ -5769,7 +6037,11 @@ def autoname_elements() -> None: Utility to simplify mass-naming of parser elements, for generating railroad diagram with named subdiagrams. """ - for name, var in sys._getframe().f_back.f_locals.items(): + calling_frame = sys._getframe().f_back + if calling_frame is None: + return + calling_frame = typing.cast(types.FrameType, calling_frame) + for name, var in calling_frame.f_locals.items(): if isinstance(var, ParserElement) and not var.customName: var.set_name(name) @@ -5783,9 +6055,28 @@ sgl_quoted_string = Combine( ).set_name("string enclosed in single quotes") quoted_string = Combine( - Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"' - | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'" -).set_name("quotedString using single or double quotes") + (Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').set_name( + "double quoted string" + ) + | (Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").set_name( + "single quoted string" + ) +).set_name("quoted string using single or double quotes") + +python_quoted_string = Combine( + (Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*', flags=re.MULTILINE) + '"""').set_name( + "multiline double quoted string" + ) + ^ ( + Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*", flags=re.MULTILINE) + "'''" + ).set_name("multiline single quoted string") + ^ (Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').set_name( + "double quoted string" + ) + ^ (Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").set_name( + "single quoted string" + ) +).set_name("Python quoted string") unicode_string = Combine("u" + quoted_string.copy()).set_name("unicode string literal") @@ -5800,9 +6091,7 @@ _builtin_exprs: List[ParserElement] = [ ] # backward compatibility names -tokenMap = token_map -conditionAsParseAction = condition_as_parse_action -nullDebugAction = null_debug_action +# fmt: off sglQuotedString = sgl_quoted_string dblQuotedString = dbl_quoted_string quotedString = quoted_string @@ -5811,4 +6100,16 @@ lineStart = line_start lineEnd = line_end stringStart = string_start stringEnd = string_end -traceParseAction = trace_parse_action + +@replaced_by_pep8(null_debug_action) +def nullDebugAction(): ... + +@replaced_by_pep8(trace_parse_action) +def traceParseAction(): ... + +@replaced_by_pep8(condition_as_parse_action) +def conditionAsParseAction(): ... + +@replaced_by_pep8(token_map) +def tokenMap(): ... +# fmt: on diff --git a/src/pip/_vendor/pyparsing/diagram/__init__.py b/src/pip/_vendor/pyparsing/diagram/__init__.py index 1506d66bf..83f9018ee 100644 --- a/src/pip/_vendor/pyparsing/diagram/__init__.py +++ b/src/pip/_vendor/pyparsing/diagram/__init__.py @@ -1,3 +1,4 @@ +# mypy: ignore-errors import railroad from pip._vendor import pyparsing import typing @@ -17,11 +18,13 @@ import inspect jinja2_template_source = """\ +{% if not embed %} +{% endif %} {% if not head %} -