From 6f1c5dc949294064c4bbc0acef57690269f7b2f7 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 13 Jul 2021 17:43:52 +0800 Subject: [PATCH] Remove pkg_resources usages from 'pip freeze' --- news/10157.process.rst | 10 +- src/pip/_internal/operations/freeze.py | 140 +++++++++++++------------ tests/functional/test_freeze.py | 7 +- 3 files changed, 85 insertions(+), 72 deletions(-) diff --git a/news/10157.process.rst b/news/10157.process.rst index 434c4d58f..f4d56c0ea 100644 --- a/news/10157.process.rst +++ b/news/10157.process.rst @@ -1,3 +1,7 @@ -``pip list`` and ``pip show`` no longer normalize underscore (``_``) in -distribution names to dash (``-``). This is done as a part of the refactoring to -prepare for the migration to ``importlib.metadata``. +``pip freeze``, ``pip list``, and ``pip show`` no longer normalize underscore +(``_``) in distribution names to dash (``-``). This is a side effect of the +migration to ``importlib.metadata``, since the underscore-dash normalization +behavior is non-standard and specific to setuptools. This should not affect +other parts of pip (for example, when feeding the ``pip freeze`` result back +into ``pip install``) since pip internally performs standard PEP 503 +normalization independently to setuptools. diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index ad2255435..defb20c7c 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -7,30 +7,32 @@ from typing import ( Iterable, Iterator, List, + NamedTuple, Optional, Set, - Tuple, Union, ) +from pip._vendor.packaging.requirements import Requirement from pip._vendor.packaging.utils import canonicalize_name -from pip._vendor.pkg_resources import Distribution, Requirement, RequirementParseError +from pip._vendor.packaging.version import Version from pip._internal.exceptions import BadCommand, InstallationError +from pip._internal.metadata import BaseDistribution, get_environment from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) from pip._internal.req.req_file import COMMENT_RE -from pip._internal.utils.direct_url_helpers import ( - direct_url_as_pep440_direct_reference, - dist_get_direct_url, -) -from pip._internal.utils.misc import dist_is_editable, get_installed_distributions +from pip._internal.utils.direct_url_helpers import direct_url_as_pep440_direct_reference logger = logging.getLogger(__name__) -RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] + +class _EditableInfo(NamedTuple): + requirement: Optional[str] + editable: bool + comments: List[str] def freeze( @@ -45,24 +47,13 @@ def freeze( # type: (...) -> Iterator[str] installations = {} # type: Dict[str, FrozenRequirement] - for dist in get_installed_distributions( - local_only=local_only, - skip=(), - user_only=user_only, - paths=paths - ): - try: - req = FrozenRequirement.from_dist(dist) - except RequirementParseError as exc: - # We include dist rather than dist.project_name because the - # dist string includes more information, like the version and - # location. We also include the exception message to aid - # troubleshooting. - logger.warning( - 'Could not generate requirement for distribution %r: %s', - dist, exc - ) - continue + dists = get_environment(paths).iter_installed_distributions( + local_only=local_only, + skip=(), + user_only=user_only, + ) + for dist in dists: + req = FrozenRequirement.from_dist(dist) if exclude_editable and req.editable: continue installations[req.canonical_name] = req @@ -160,49 +151,68 @@ def freeze( yield str(installation).rstrip() -def get_requirement_info(dist): - # type: (Distribution) -> RequirementInfo +def _format_as_name_version(dist: BaseDistribution) -> str: + if isinstance(dist.version, Version): + return f"{dist.raw_name}=={dist.version}" + return f"{dist.raw_name}==={dist.version}" + + +def _get_editable_info(dist: BaseDistribution) -> _EditableInfo: """ Compute and return values (req, editable, comments) for use in FrozenRequirement.from_dist(). """ - if not dist_is_editable(dist): - return (None, False, []) + if not dist.editable: + return _EditableInfo(requirement=None, editable=False, comments=[]) + if dist.location is None: + display = _format_as_name_version(dist) + logger.warning("Editable requirement not found on disk: %s", display) + return _EditableInfo( + requirement=None, + editable=True, + comments=[f"# Editable install not found ({display})"], + ) location = os.path.normcase(os.path.abspath(dist.location)) from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs + vcs_backend = vcs.get_backend_for_dir(location) if vcs_backend is None: - req = dist.as_requirement() + display = _format_as_name_version(dist) logger.debug( - 'No VCS found for editable requirement "%s" in: %r', req, + 'No VCS found for editable requirement "%s" in: %r', display, location, ) - comments = [ - f'# Editable install with no version control ({req})' - ] - return (location, True, comments) + return _EditableInfo( + requirement=location, + editable=True, + comments=[f'# Editable install with no version control ({display})'], + ) + + vcs_name = type(vcs_backend).__name__ try: - req = vcs_backend.get_src_requirement(location, dist.project_name) + req = vcs_backend.get_src_requirement(location, dist.raw_name) except RemoteNotFoundError: - req = dist.as_requirement() - comments = [ - '# Editable {} install with no remote ({})'.format( - type(vcs_backend).__name__, req, - ) - ] - return (location, True, comments) + display = _format_as_name_version(dist) + return _EditableInfo( + requirement=location, + editable=True, + comments=[f'# Editable {vcs_name} install with no remote ({display})'], + ) except RemoteNotValidError as ex: - req = dist.as_requirement() - comments = [ - f"# Editable {type(vcs_backend).__name__} install ({req}) with " - f"either a deleted local remote or invalid URI:", - f"# '{ex.url}'", - ] - return (location, True, comments) + display = _format_as_name_version(dist) + return _EditableInfo( + requirement=location, + editable=True, + comments=[ + f"# Editable {vcs_name} install ({display}) with either a deleted " + f"local remote or invalid URI:", + f"# '{ex.url}'", + ], + ) except BadCommand: logger.warning( @@ -211,7 +221,7 @@ def get_requirement_info(dist): location, vcs_backend.name, ) - return (None, True, []) + return _EditableInfo(requirement=None, editable=True, comments=[]) except InstallationError as exc: logger.warning( @@ -219,14 +229,15 @@ def get_requirement_info(dist): "falling back to uneditable format", exc ) else: - return (req, True, []) + return _EditableInfo(requirement=req, editable=True, comments=[]) - logger.warning( - 'Could not determine repository location of %s', location + logger.warning('Could not determine repository location of %s', location) + + return _EditableInfo( + requirement=None, + editable=False, + comments=['## !! Could not determine repository location'], ) - comments = ['## !! Could not determine repository location'] - - return (None, False, comments) class FrozenRequirement: @@ -239,25 +250,24 @@ class FrozenRequirement: self.comments = comments @classmethod - def from_dist(cls, dist): - # type: (Distribution) -> FrozenRequirement + def from_dist(cls, dist: BaseDistribution) -> "FrozenRequirement": # TODO `get_requirement_info` is taking care of editable requirements. # TODO This should be refactored when we will add detection of # editable that provide .dist-info metadata. - req, editable, comments = get_requirement_info(dist) + req, editable, comments = _get_editable_info(dist) if req is None and not editable: # if PEP 610 metadata is present, attempt to use it - direct_url = dist_get_direct_url(dist) + direct_url = dist.direct_url if direct_url: req = direct_url_as_pep440_direct_reference( - direct_url, dist.project_name + direct_url, dist.raw_name ) comments = [] if req is None: # name==version requirement - req = dist.as_requirement() + req = _format_as_name_version(dist) - return cls(dist.project_name, req, editable, comments=comments) + return cls(dist.raw_name, req, editable, comments=comments) def __str__(self): # type: () -> str diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 1a2589df9..6a0f4c8e5 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -6,7 +6,6 @@ from doctest import ELLIPSIS, OutputChecker import pytest from pip._vendor.packaging.utils import canonicalize_name -from pip._vendor.pkg_resources import safe_name from tests.lib import ( _create_test_package, @@ -88,9 +87,9 @@ def test_exclude_and_normalization(script, tmpdir): name="Normalizable_Name", version="1.0").save_to_dir(tmpdir) script.pip("install", "--no-index", req_path) result = script.pip("freeze") - assert "Normalizable-Name" in result.stdout + assert "Normalizable_Name" in result.stdout result = script.pip("freeze", "--exclude", "normalizablE-namE") - assert "Normalizable-Name" not in result.stdout + assert "Normalizable_Name" not in result.stdout def test_freeze_multiple_exclude_with_all(script, with_wheel): @@ -136,7 +135,7 @@ def test_freeze_with_invalid_names(script): # Check all valid names are in the output. output_lines = {line.strip() for line in result.stdout.splitlines()} for name in valid_pkgnames: - assert f"{safe_name(name)}==1.0" in output_lines + assert f"{name}==1.0" in output_lines # Check all invalid names are excluded from the output. canonical_invalid_names = {canonicalize_name(n) for n in invalid_pkgnames}