McSinyx
/
pip
mirror of https://github.com/pypa/pip
1
1
Fork 0
Code Issues Releases Activity

Merge 227d8e8dd2 into a15dd75d98

This commit is contained in:
Danny McClanahan 2023-11-29 21:12:11 +08:00 committed by GitHub
parent a15dd75d98 227d8e8dd2
commit 8bbeb829cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 924 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
news/12186.bugfix.rst Normal file
View File

@ -0,0 +1 @@
Avoid downloading any dists in ``install --dry-run`` if PEP 658 ``.metadata`` files or lazy wheels are available.

1
news/12256.feature.rst Normal file
View File

@ -0,0 +1 @@
Cache computed metadata from sdists and lazy wheels in ``~/.cache/pip/link-metadata``.

120
src/pip/_internal/cache.py
View File

@ -1,12 +1,14 @@
"""Cache Management
"""
import abc
import hashlib
import json
import logging
import os
import re
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Dict, Iterator, List, Optional, Tuple
from pip._vendor.packaging.tags import Tag, interpreter_name, interpreter_version
from pip._vendor.packaging.utils import canonicalize_name
@ -15,21 +17,71 @@ from pip._internal.exceptions import InvalidWheelFilename
from pip._internal.models.direct_url import DirectUrl
from pip._internal.models.link import Link
from pip._internal.models.wheel import Wheel
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
from pip._internal.utils.urls import path_to_url
from pip._internal.vcs import vcs
logger = logging.getLogger(__name__)
_egg_info_re = re.compile(r"([a-z0-9_.]+)-([a-z0-9_.!+-]+)", re.IGNORECASE)
ORIGIN_JSON_NAME = "origin.json"
def _contains_egg_info(s: str) -> bool:
"""Determine whether the string looks like an egg_info.
:param s: The string to parse. E.g. foo-2.1
"""
return bool(_egg_info_re.search(s))
def should_cache(
req: InstallRequirement,
) -> bool:
"""
Return whether a built InstallRequirement can be stored in the persistent
wheel cache, assuming the wheel cache is available, and _should_build()
has determined a wheel needs to be built.
"""
if not req.link:
return False
if req.link.is_wheel:
return False
if req.editable or not req.source_dir:
# never cache editable requirements
return False
if req.link and req.link.is_vcs:
# VCS checkout. Do not cache
# unless it points to an immutable commit hash.
assert not req.editable
assert req.source_dir
vcs_backend = vcs.get_backend_for_scheme(req.link.scheme)
assert vcs_backend
if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir):
return True
return False
assert req.link
base, ext = req.link.splitext()
if _contains_egg_info(base):
return True
# Otherwise, do not cache.
return False
def _hash_dict(d: Dict[str, str]) -> str:
"""Return a stable sha224 of a dictionary."""
s = json.dumps(d, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
return hashlib.sha224(s.encode("ascii")).hexdigest()
class Cache:
class Cache(abc.ABC):
"""An abstract class - provides cache directories for data from links
:param cache_dir: The root of the cache.
@ -73,20 +125,28 @@ class Cache:
return parts
def _get_candidates(self, link: Link, canonical_package_name: str) -> List[Any]:
can_not_cache = not self.cache_dir or not canonical_package_name or not link
if can_not_cache:
return []
path = self.get_path_for_link(link)
if os.path.isdir(path):
return [(candidate, path) for candidate in os.listdir(path)]
return []
@abc.abstractmethod
def get_path_for_link(self, link: Link) -> str:
"""Return a directory to store cached items in for link."""
raise NotImplementedError()
...
def cache_path(self, link: Link) -> Path:
return Path(self.get_path_for_link(link))
class LinkMetadataCache(Cache):
"""Persistently store the metadata of dists found at each link."""
def get_path_for_link(self, link: Link) -> str:
parts = self._get_cache_path_parts(link)
assert self.cache_dir
return os.path.join(self.cache_dir, "link-metadata", *parts)
class WheelCacheBase(Cache):
"""Specializations to the cache concept for wheels."""
@abc.abstractmethod
def get(
self,
link: Link,
@ -96,10 +156,27 @@ class Cache:
"""Returns a link to a cached item if it exists, otherwise returns the
passed link.
"""
raise NotImplementedError()
...
def _can_cache(self, link: Link, canonical_package_name: str) -> bool:
return bool(self.cache_dir and canonical_package_name and link)
def _get_candidates(
self, link: Link, canonical_package_name: str
) -> Iterator[Tuple[str, str]]:
if not self._can_cache(link, canonical_package_name):
return
path = self.get_path_for_link(link)
if not os.path.isdir(path):
return
for candidate in os.scandir(path):
if candidate.is_file():
yield (candidate.name, path)
class SimpleWheelCache(Cache):
class SimpleWheelCache(WheelCacheBase):
"""A cache of wheels for future installs."""
def __init__(self, cache_dir: str) -> None:
@ -131,7 +208,7 @@ class SimpleWheelCache(Cache):
package_name: Optional[str],
supported_tags: List[Tag],
) -> Link:
candidates = []
candidates: List[Tuple[int, str, str]] = []
if not package_name:
return link
@ -205,7 +282,7 @@ class CacheEntry:
)
class WheelCache(Cache):
class WheelCache(WheelCacheBase):
"""Wraps EphemWheelCache and SimpleWheelCache into a single Cache
This Cache allows for gracefully degradation, using the ephem wheel cache
@ -223,6 +300,15 @@ class WheelCache(Cache):
def get_ephem_path_for_link(self, link: Link) -> str:
return self._ephem_cache.get_path_for_link(link)
def resolve_cache_dir(self, req: InstallRequirement) -> str:
"""Return the persistent or temporary cache directory where the built or
downloaded wheel should be stored."""
cache_available = bool(self.cache_dir)
assert req.link, req
if cache_available and should_cache(req):
return self.get_path_for_link(req.link)
return self.get_ephem_path_for_link(req.link)
def get(
self,
link: Link,

7
src/pip/_internal/cli/req_command.py
View File

@ -12,7 +12,7 @@ from functools import partial
from optparse import Values
from typing import TYPE_CHECKING, Any, List, Optional, Tuple
from pip._internal.cache import WheelCache
from pip._internal.cache import LinkMetadataCache, WheelCache
from pip._internal.cli import cmdoptions
from pip._internal.cli.base_command import Command
from pip._internal.cli.command_context import CommandContextMixIn
@ -305,6 +305,10 @@ class RequirementCommand(IndexGroupCommand):
"fast-deps has no effect when used with the legacy resolver."
)
if options.cache_dir:
metadata_cache = LinkMetadataCache(options.cache_dir)
else:
metadata_cache = None
return RequirementPreparer(
build_dir=temp_build_dir_path,
src_dir=options.src_dir,
@ -320,6 +324,7 @@ class RequirementCommand(IndexGroupCommand):
lazy_wheel=lazy_wheel,
verbosity=verbosity,
legacy_resolver=legacy_resolver,
metadata_cache=metadata_cache,
)
@classmethod

4
src/pip/_internal/commands/download.py
View File

@ -130,6 +130,9 @@ class DownloadCommand(RequirementCommand):
self.trace_basic_info(finder)
requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
preparer.finalize_linked_requirements(
requirement_set.requirements.values(), hydrate_virtual_reqs=True
)
downloaded: List[str] = []
for req in requirement_set.requirements.values():
@ -138,7 +141,6 @@ class DownloadCommand(RequirementCommand):
preparer.save_linked_requirement(req)
downloaded.append(req.name)
preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
requirement_set.warn_legacy_versions_and_specifiers()
if downloaded:

7
src/pip/_internal/commands/install.py
View File

@ -84,7 +84,8 @@ class InstallCommand(RequirementCommand):
help=(
"Don't actually install anything, just print what would be. "
"Can be used in combination with --ignore-installed "
"to 'resolve' the requirements."
"to 'resolve' the requirements. If PEP 658 or fast-deps metadata is "
"available, --dry-run also avoids downloading the dependency at all."
),
)
self.cmd_opts.add_option(
@ -377,6 +378,10 @@ class InstallCommand(RequirementCommand):
requirement_set = resolver.resolve(
reqs, check_supported_wheels=not options.target_dir
)
preparer.finalize_linked_requirements(
requirement_set.requirements.values(),
hydrate_virtual_reqs=not options.dry_run,
)
if options.json_report_file:
report = InstallationReport(requirement_set.requirements_to_install)

4
src/pip/_internal/commands/wheel.py
View File

@ -145,6 +145,9 @@ class WheelCommand(RequirementCommand):
self.trace_basic_info(finder)
requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
preparer.finalize_linked_requirements(
requirement_set.requirements.values(), hydrate_virtual_reqs=True
)
reqs_to_build: List[InstallRequirement] = []
for req in requirement_set.requirements.values():
@ -153,7 +156,6 @@ class WheelCommand(RequirementCommand):
elif should_build_for_wheel_command(req):
reqs_to_build.append(req)
preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
requirement_set.warn_legacy_versions_and_specifiers()
# build wheels

5
src/pip/_internal/distributions/__init__.py
View File

@ -1,4 +1,5 @@
from pip._internal.distributions.base import AbstractDistribution
from pip._internal.distributions.installed import InstalledDistribution
from pip._internal.distributions.sdist import SourceDistribution
from pip._internal.distributions.wheel import WheelDistribution
from pip._internal.req.req_install import InstallRequirement
@ -8,6 +9,10 @@ def make_distribution_for_install_requirement(
install_req: InstallRequirement,
) -> AbstractDistribution:
"""Returns a Distribution for the given InstallRequirement"""
# Only pre-installed requirements will have a .satisfied_by dist.
if install_req.satisfied_by:
return InstalledDistribution(install_req)
# Editable requirements will always be source distributions. They use the
# legacy logic until we create a modern standard for them.
if install_req.editable:

19
src/pip/_internal/distributions/base.py
View File

@ -35,11 +35,17 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
If None, then this dist has no work to do in the build tracker, and
``.prepare_distribution_metadata()`` will not be called."""
raise NotImplementedError()
...
@abc.abstractmethod
def get_metadata_distribution(self) -> BaseDistribution:
raise NotImplementedError()
"""Generate a concrete ``BaseDistribution`` instance for this artifact.
The implementation should also cache the result with
``self.req.cache_concrete_dist()`` so the distribution is available to other
users of the ``InstallRequirement``. This method is not called within the build
tracker context, so it should not identify any new setup requirements."""
...
@abc.abstractmethod
def prepare_distribution_metadata(
@ -48,4 +54,11 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
build_isolation: bool,
check_build_deps: bool,
) -> None:
raise NotImplementedError()
"""Generate the information necessary to extract metadata from the artifact.
This method will be executed within the context of ``BuildTracker#track()``, so
it needs to fully identify any seutp requirements so they can be added to the
same active set of tracked builds, while ``.get_metadata_distribution()`` takes
care of generating and caching the ``BaseDistribution`` to expose to the rest of
the resolve."""
...

6
src/pip/_internal/distributions/installed.py
View File

@ -17,8 +17,10 @@ class InstalledDistribution(AbstractDistribution):
return None
def get_metadata_distribution(self) -> BaseDistribution:
assert self.req.satisfied_by is not None, "not actually installed"
return self.req.satisfied_by
dist = self.req.satisfied_by
assert dist is not None, "not actually installed"
self.req.cache_concrete_dist(dist)
return dist
def prepare_distribution_metadata(
self,

20
src/pip/_internal/distributions/sdist.py
View File

@ -1,11 +1,11 @@
import logging
from typing import Iterable, Optional, Set, Tuple
from typing import Iterable, Set, Tuple
from pip._internal.build_env import BuildEnvironment
from pip._internal.distributions.base import AbstractDistribution
from pip._internal.exceptions import InstallationError
from pip._internal.index.package_finder import PackageFinder
from pip._internal.metadata import BaseDistribution
from pip._internal.metadata import BaseDistribution, get_directory_distribution
from pip._internal.utils.subprocess import runner_with_spinner_message
logger = logging.getLogger(__name__)
@ -19,13 +19,19 @@ class SourceDistribution(AbstractDistribution):
"""
@property
def build_tracker_id(self) -> Optional[str]:
def build_tracker_id(self) -> str:
"""Identify this requirement uniquely by its link."""
assert self.req.link
return self.req.link.url_without_fragment
def get_metadata_distribution(self) -> BaseDistribution:
return self.req.get_dist()
assert (
self.req.metadata_directory
), "Set as part of .prepare_distribution_metadata()"
dist = get_directory_distribution(self.req.metadata_directory)
self.req.cache_concrete_dist(dist)
self.req.validate_sdist_metadata()
return dist
def prepare_distribution_metadata(
self,
@ -64,7 +70,11 @@ class SourceDistribution(AbstractDistribution):
self._raise_conflicts("the backend dependencies", conflicting)
if missing:
self._raise_missing_reqs(missing)
self.req.prepare_metadata()
# NB: we must still call .cache_concrete_dist() and .validate_sdist_metadata()
# before the InstallRequirement itself has been updated with the metadata from
# this directory!
self.req.prepare_metadata_directory()
def _prepare_build_backend(self, finder: PackageFinder) -> None:
# Isolate in a BuildEnvironment and install the build-time

4
src/pip/_internal/distributions/wheel.py
View File

@ -29,7 +29,9 @@ class WheelDistribution(AbstractDistribution):
assert self.req.local_file_path, "Set as part of preparation during download"
assert self.req.name, "Wheels are never unnamed"
wheel = FilesystemWheel(self.req.local_file_path)
return get_wheel_distribution(wheel, canonicalize_name(self.req.name))
dist = get_wheel_distribution(wheel, canonicalize_name(self.req.name))
self.req.cache_concrete_dist(dist)
return dist
def prepare_distribution_metadata(
self,

19
src/pip/_internal/exceptions.py
View File

@ -250,6 +250,25 @@ class NoneMetadataError(PipError):
return f"None {self.metadata_name} metadata found for distribution: {self.dist}"
class CacheMetadataError(PipError):
"""Raised when de/serializing a requirement into the metadata cache."""
def __init__(
self,
req: "InstallRequirement",
reason: str,
) -> None:
"""
:param req: The requirement we attempted to cache.
:param reason: Context about the precise error that occurred.
"""
self.req = req
self.reason = reason
def __str__(self) -> str:
return f"{self.reason} for {self.req} from {self.req.link}"
class UserInstallationInvalid(InstallationError):
"""A --user install is requested on an environment without user site."""

10
src/pip/_internal/metadata/__init__.py
View File

@ -6,7 +6,14 @@ from typing import TYPE_CHECKING, List, Optional, Type, cast
from pip._internal.utils.misc import strtobool
from .base import BaseDistribution, BaseEnvironment, FilesystemWheel, MemoryWheel, Wheel
from .base import (
BaseDistribution,
BaseEnvironment,
FilesystemWheel,
MemoryWheel,
Wheel,
serialize_metadata,
)
if TYPE_CHECKING:
from typing import Literal, Protocol
@ -23,6 +30,7 @@ __all__ = [
"get_environment",
"get_wheel_distribution",
"select_backend",
"serialize_metadata",
]

36
src/pip/_internal/metadata/base.py
View File

@ -1,6 +1,9 @@
import csv
import email.generator
import email.message
import email.policy
import functools
import io
import json
import logging
import pathlib
@ -97,6 +100,18 @@ def _convert_installed_files_path(
return str(pathlib.Path(*info, *entry))
def serialize_metadata(msg: email.message.Message) -> str:
"""Write a dist's metadata to a string.
Calling ``str(dist.metadata)`` may raise an error by misinterpreting RST directives
as email headers. This method uses the more robust ``email.policy.EmailPolicy`` to
avoid those parsing errors."""
out = io.StringIO()
g = email.generator.Generator(out, policy=email.policy.EmailPolicy())
g.flatten(msg)
return out.getvalue()
class RequiresEntry(NamedTuple):
requirement: str
extra: str
@ -104,6 +119,15 @@ class RequiresEntry(NamedTuple):
class BaseDistribution(Protocol):
@property
def is_concrete(self) -> bool:
"""Whether the distribution really exists somewhere on disk.
If this is false, it has been synthesized from metadata, e.g. via
``.from_metadata_file_contents()``, or ``.from_wheel()`` against
a ``MemoryWheel``."""
raise NotImplementedError()
@classmethod
def from_directory(cls, directory: str) -> "BaseDistribution":
"""Load the distribution from a metadata directory.
@ -681,6 +705,10 @@ class BaseEnvironment:
class Wheel(Protocol):
location: str
@property
def is_concrete(self) -> bool:
raise NotImplementedError()
def as_zipfile(self) -> zipfile.ZipFile:
raise NotImplementedError()
@ -689,6 +717,10 @@ class FilesystemWheel(Wheel):
def __init__(self, location: str) -> None:
self.location = location
@property
def is_concrete(self) -> bool:
return True
def as_zipfile(self) -> zipfile.ZipFile:
return zipfile.ZipFile(self.location, allowZip64=True)
@ -698,5 +730,9 @@ class MemoryWheel(Wheel):
self.location = location
self.stream = stream
@property
def is_concrete(self) -> bool:
return False
def as_zipfile(self) -> zipfile.ZipFile:
return zipfile.ZipFile(self.stream, allowZip64=True)

17
src/pip/_internal/metadata/importlib/_dists.py
View File

@ -98,16 +98,22 @@ class Distribution(BaseDistribution):
dist: importlib.metadata.Distribution,
info_location: Optional[BasePath],
installed_location: Optional[BasePath],
concrete: bool,
) -> None:
self._dist = dist
self._info_location = info_location
self._installed_location = installed_location
self._concrete = concrete
@property
def is_concrete(self) -> bool:
return self._concrete
@classmethod
def from_directory(cls, directory: str) -> BaseDistribution:
info_location = pathlib.Path(directory)
dist = importlib.metadata.Distribution.at(info_location)
return cls(dist, info_location, info_location.parent)
return cls(dist, info_location, info_location.parent, concrete=True)
@classmethod
def from_metadata_file_contents(
@ -124,7 +130,7 @@ class Distribution(BaseDistribution):
metadata_path.write_bytes(metadata_contents)
# Construct dist pointing to the newly created directory.
dist = importlib.metadata.Distribution.at(metadata_path.parent)
return cls(dist, metadata_path.parent, None)
return cls(dist, metadata_path.parent, None, concrete=False)
@classmethod
def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
@ -135,7 +141,12 @@ class Distribution(BaseDistribution):
raise InvalidWheel(wheel.location, name) from e
except UnsupportedWheel as e:
raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
return cls(dist, dist.info_location, pathlib.PurePosixPath(wheel.location))
return cls(
dist,
dist.info_location,
pathlib.PurePosixPath(wheel.location),
concrete=wheel.is_concrete,
)
@property
def location(self) -> Optional[str]:

8
src/pip/_internal/metadata/importlib/_envs.py
View File

@ -81,7 +81,7 @@ class _DistributionFinder:
installed_location: Optional[BasePath] = None
else:
installed_location = info_location.parent
yield Distribution(dist, info_location, installed_location)
yield Distribution(dist, info_location, installed_location, concrete=True)
def find_linked(self, location: str) -> Iterator[BaseDistribution]:
"""Read location in egg-link files and return distributions in there.
@ -105,7 +105,7 @@ class _DistributionFinder:
continue
target_location = str(path.joinpath(target_rel))
for dist, info_location in self._find_impl(target_location):
yield Distribution(dist, info_location, path)
yield Distribution(dist, info_location, path, concrete=True)
def _find_eggs_in_dir(self, location: str) -> Iterator[BaseDistribution]:
from pip._vendor.pkg_resources import find_distributions
@ -117,7 +117,7 @@ class _DistributionFinder:
if not entry.name.endswith(".egg"):
continue
for dist in find_distributions(entry.path):
yield legacy.Distribution(dist)
yield legacy.Distribution(dist, concrete=True)
def _find_eggs_in_zip(self, location: str) -> Iterator[BaseDistribution]:
from pip._vendor.pkg_resources import find_eggs_in_zip
@ -129,7 +129,7 @@ class _DistributionFinder:
except zipimport.ZipImportError:
return
for dist in find_eggs_in_zip(importer, location):
yield legacy.Distribution(dist)
yield legacy.Distribution(dist, concrete=True)
def find_eggs(self, location: str) -> Iterator[BaseDistribution]:
"""Find eggs in a location.

15
src/pip/_internal/metadata/pkg_resources.py
View File

@ -73,8 +73,13 @@ class InMemoryMetadata:
class Distribution(BaseDistribution):
def __init__(self, dist: pkg_resources.Distribution) -> None:
def __init__(self, dist: pkg_resources.Distribution, concrete: bool) -> None:
self._dist = dist
self._concrete = concrete
@property
def is_concrete(self) -> bool:
return self._concrete
@classmethod
def from_directory(cls, directory: str) -> BaseDistribution:
@ -94,7 +99,7 @@ class Distribution(BaseDistribution):
dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
dist = dist_cls(base_dir, project_name=dist_name, metadata=metadata)
return cls(dist)
return cls(dist, concrete=True)
@classmethod
def from_metadata_file_contents(
@ -111,7 +116,7 @@ class Distribution(BaseDistribution):
metadata=InMemoryMetadata(metadata_dict, filename),
project_name=project_name,
)
return cls(dist)
return cls(dist, concrete=False)
@classmethod
def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
@ -132,7 +137,7 @@ class Distribution(BaseDistribution):
metadata=InMemoryMetadata(metadata_dict, wheel.location),
project_name=name,
)
return cls(dist)
return cls(dist, concrete=wheel.is_concrete)
@property
def location(self) -> Optional[str]:
@ -241,7 +246,7 @@ class Environment(BaseEnvironment):
def _iter_distributions(self) -> Iterator[BaseDistribution]:
for dist in self._ws:
yield Distribution(dist)
yield Distribution(dist, concrete=True)
def _search_distribution(self, name: str) -> Optional[BaseDistribution]:
"""Find a distribution matching the ``name`` in the environment.

2
src/pip/_internal/models/installation_report.py
View File

@ -32,7 +32,7 @@ class InstallationReport:
"requested": ireq.user_supplied,
# PEP 566 json encoding for metadata
# https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
"metadata": ireq.get_dist().metadata_dict,
"metadata": ireq.cached_dist.metadata_dict,
}
if ireq.user_supplied and ireq.extras:
# For top level requirements, the list of requested extras, if any.

2
src/pip/_internal/network/download.py
View File

@ -113,7 +113,7 @@ def _get_http_response_filename(resp: Response, link: Link) -> str:
def _http_get_download(session: PipSession, link: Link) -> Response:
target_url = link.url.split("#", 1)[0]
target_url = link.url_without_fragment
resp = session.get(target_url, headers=HEADERS, stream=True)
raise_for_status(resp)
return resp

5
src/pip/_internal/operations/check.py
View File

@ -9,7 +9,6 @@ from pip._vendor.packaging.specifiers import LegacySpecifier
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import LegacyVersion
from pip._internal.distributions import make_distribution_for_install_requirement
from pip._internal.metadata import get_default_environment
from pip._internal.metadata.base import DistributionVersion
from pip._internal.req.req_install import InstallRequirement
@ -127,8 +126,8 @@ def _simulate_installation_of(
# Modify it as installing requirement_set would (assuming no errors)
for inst_req in to_install:
abstract_dist = make_distribution_for_install_requirement(inst_req)
dist = abstract_dist.get_metadata_distribution()
assert inst_req.is_concrete
dist = inst_req.cached_dist
name = dist.canonical_name
package_set[name] = PackageDetails(dist.version, list(dist.iter_dependencies()))

332
src/pip/_internal/operations/prepare.py
View File

@ -4,17 +4,22 @@
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import gzip
import json
import mimetypes
import os
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from typing import Dict, Iterable, List, Optional, Tuple
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.requests.exceptions import InvalidSchema
from pip._internal.cache import LinkMetadataCache, should_cache
from pip._internal.distributions import make_distribution_for_install_requirement
from pip._internal.distributions.installed import InstalledDistribution
from pip._internal.exceptions import (
CacheMetadataError,
DirectoryUrlHashUnsupported,
HashMismatch,
HashUnpinned,
@ -24,7 +29,11 @@ from pip._internal.exceptions import (
VcsHashUnsupported,
)
from pip._internal.index.package_finder import PackageFinder
from pip._internal.metadata import BaseDistribution, get_metadata_distribution
from pip._internal.metadata import (
BaseDistribution,
get_metadata_distribution,
serialize_metadata,
)
from pip._internal.models.direct_url import ArchiveInfo
from pip._internal.models.link import Link
from pip._internal.models.wheel import Wheel
@ -62,16 +71,17 @@ def _get_prepared_distribution(
finder: PackageFinder,
build_isolation: bool,
check_build_deps: bool,
) -> BaseDistribution:
) -> Tuple[bool, BaseDistribution]:
"""Prepare a distribution for installation."""
abstract_dist = make_distribution_for_install_requirement(req)
tracker_id = abstract_dist.build_tracker_id
if tracker_id is not None:
builds_metadata = tracker_id is not None
if builds_metadata:
with build_tracker.track(req, tracker_id):
abstract_dist.prepare_distribution_metadata(
finder, build_isolation, check_build_deps
)
return abstract_dist.get_metadata_distribution()
return (builds_metadata, abstract_dist.get_metadata_distribution())
def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
@ -188,6 +198,8 @@ def _check_download_dir(
) -> Optional[str]:
"""Check download_dir for previously downloaded file with correct hash
If a correct file is found return its path else None
If a file is found at the given path, but with an invalid hash, the file is deleted.
"""
download_path = os.path.join(download_dir, link.filename)
@ -210,10 +222,49 @@ def _check_download_dir(
return download_path
@dataclass(frozen=True)
class CacheableDist:
metadata: str
filename: Path
canonical_name: str
@classmethod
def from_dist(cls, link: Link, dist: BaseDistribution) -> "CacheableDist":
"""Extract the serializable data necessary to generate a metadata-only dist."""
return cls(
metadata=serialize_metadata(dist.metadata),
filename=Path(link.filename),
canonical_name=dist.canonical_name,
)
def to_dist(self) -> BaseDistribution:
"""Return a metadata-only dist from the deserialized cache entry."""
return get_metadata_distribution(
metadata_contents=self.metadata.encode("utf-8"),
filename=str(self.filename),
canonical_name=self.canonical_name,
)
def to_json(self) -> Dict[str, str]:
return {
"metadata": self.metadata,
"filename": str(self.filename),
"canonical_name": self.canonical_name,
}
@classmethod
def from_json(cls, args: Dict[str, str]) -> "CacheableDist":
return cls(
metadata=args["metadata"],
filename=Path(args["filename"]),
canonical_name=args["canonical_name"],
)
class RequirementPreparer:
"""Prepares a Requirement"""
def __init__(
def __init__( # noqa: PLR0913
self,
build_dir: str,
download_dir: Optional[str],
@ -229,6 +280,7 @@ class RequirementPreparer:
lazy_wheel: bool,
verbosity: int,
legacy_resolver: bool,
metadata_cache: Optional[LinkMetadataCache] = None,
) -> None:
super().__init__()
@ -271,6 +323,8 @@ class RequirementPreparer:
# Previous "header" printed for a link-based InstallRequirement
self._previous_requirement_header = ("", "")
self._metadata_cache = metadata_cache
def _log_preparing_link(self, req: InstallRequirement) -> None:
"""Provide context for the requirement being prepared."""
if req.link.is_file and not req.is_wheel_from_cache:
@ -363,14 +417,81 @@ class RequirementPreparer:
)
return None
if self.require_hashes:
# Hash checking also means hashes are provided for all reqs, so no resolve
# is necessary and metadata-only fetching provides no speedup.
logger.debug(
"Metadata-only fetching is not used as hash checking is required",
)
return None
# Try PEP 658 metadata first, then fall back to lazy wheel if unavailable.
return self._fetch_metadata_using_link_data_attr(
req
) or self._fetch_metadata_using_lazy_wheel(req.link)
return (
self._fetch_cached_metadata(req)
or self._fetch_metadata_using_link_data_attr(req)
or self._fetch_metadata_using_lazy_wheel(req)
)
def _locate_metadata_cache_entry(self, link: Link) -> Optional[Path]:
"""If the metadata cache is active, generate a filesystem path from the hash of
the given Link."""
if self._metadata_cache is None:
return None
return self._metadata_cache.cache_path(link)
def _fetch_cached_metadata(
self, req: InstallRequirement
) -> Optional[BaseDistribution]:
cached_path = self._locate_metadata_cache_entry(req.link)
if cached_path is None:
return None
# Quietly continue if the cache entry does not exist.
if not os.path.isfile(cached_path):
logger.debug(
"no cached metadata for link %s at %s",
req.link,
cached_path,
)
return None
try:
with gzip.open(cached_path, mode="rt", encoding="utf-8") as f:
logger.debug(
"found cached metadata for link %s at %s", req.link, f.name
)
args = json.load(f)
cached_dist = CacheableDist.from_json(args)
return cached_dist.to_dist()
except Exception:
raise CacheMetadataError(req, "error reading cached metadata")
def _cache_metadata(
self,
req: InstallRequirement,
metadata_dist: BaseDistribution,
) -> None:
cached_path = self._locate_metadata_cache_entry(req.link)
if cached_path is None:
return
# The cache file exists already, so we have nothing to do.
if os.path.isfile(cached_path):
logger.debug(
"metadata for link %s is already cached at %s", req.link, cached_path
)
return
# The metadata cache is split across several subdirectories, so ensure the
# containing directory for the cache file exists before writing.
os.makedirs(str(cached_path.parent), exist_ok=True)
try:
cacheable_dist = CacheableDist.from_dist(req.link, metadata_dist)
args = cacheable_dist.to_json()
logger.debug("caching metadata for link %s at %s", req.link, cached_path)
with gzip.open(cached_path, mode="wt", encoding="utf-8") as f:
json.dump(args, f)
except Exception:
raise CacheMetadataError(req, "failed to serialize metadata")
def _fetch_metadata_using_link_data_attr(
self,
@ -388,6 +509,9 @@ class RequirementPreparer:
metadata_link,
)
# (2) Download the contents of the METADATA file, separate from the dist itself.
# NB: this request will hit the CacheControl HTTP cache, which will be very
# quick since the METADATA file is very small. Therefore, we can rely on
# HTTP caching instead of LinkMetadataCache.
metadata_file = get_http_url(
metadata_link,
self._download,
@ -415,33 +539,45 @@ class RequirementPreparer:
def _fetch_metadata_using_lazy_wheel(
self,
link: Link,
req: InstallRequirement,
) -> Optional[BaseDistribution]:
"""Fetch metadata using lazy wheel, if possible."""
# --use-feature=fast-deps must be provided.
if not self.use_lazy_wheel:
return None
if link.is_file or not link.is_wheel:
if req.link.is_file or not req.link.is_wheel:
logger.debug(
"Lazy wheel is not used as %r does not point to a remote wheel",
link,
req.link,
)
return None
wheel = Wheel(link.filename)
wheel = Wheel(req.link.filename)
name = canonicalize_name(wheel.name)
logger.info(
"Obtaining dependency information from %s %s",
name,
wheel.version,
)
url = link.url.split("#", 1)[0]
try:
return dist_from_wheel_url(name, url, self._session)
lazy_wheel_dist = dist_from_wheel_url(
name, req.link.url_without_fragment, self._session
)
except HTTPRangeRequestUnsupported:
logger.debug("%s does not support range requests", url)
logger.debug("%s does not support range requests", req.link)
return None
# If we've used the lazy wheel approach, then PEP 658 metadata is not available.
# If the wheel is very large (>1GB), then retrieving it from the CacheControl
# HTTP cache may take multiple seconds, even on a fast computer, and the
# preparer will unnecessarily copy the cached response to disk before deleting
# it at the end of the run. Caching the dist metadata in LinkMetadataCache means
# later pip executions can retrieve metadata within milliseconds and avoid
# thrashing the disk.
self._cache_metadata(req, lazy_wheel_dist)
return lazy_wheel_dist
def _complete_partial_requirements(
self,
partially_downloaded_reqs: Iterable[InstallRequirement],
@ -458,7 +594,21 @@ class RequirementPreparer:
links_to_fully_download: Dict[Link, InstallRequirement] = {}
for req in partially_downloaded_reqs:
assert req.link
links_to_fully_download[req.link] = req
# (1) File URLs don't need to be downloaded, so skip them.
if req.link.scheme == "file":
continue
# (2) If this is e.g. a git url, we don't know how to handle that in the
# BatchDownloader, so leave it for self._prepare_linked_requirement() at
# the end of this method, which knows how to handle any URL.
can_simply_download = True
try:
# This will raise InvalidSchema if our Session can't download it.
self._session.get_adapter(req.link.url)
except InvalidSchema:
can_simply_download = False
if can_simply_download:
links_to_fully_download[req.link] = req
batch_download = self._batch_download(
links_to_fully_download.keys(),
@ -518,41 +668,92 @@ class RequirementPreparer:
# The file is not available, attempt to fetch only metadata
metadata_dist = self._fetch_metadata_only(req)
if metadata_dist is not None:
req.needs_more_preparation = True
# These reqs now have the dependency information from the downloaded
# metadata, without having downloaded the actual dist at all.
req.cache_virtual_metadata_only_dist(metadata_dist)
return metadata_dist
# None of the optimizations worked, fully prepare the requirement
return self._prepare_linked_requirement(req, parallel_builds)
def prepare_linked_requirements_more(
self, reqs: Iterable[InstallRequirement], parallel_builds: bool = False
) -> None:
"""Prepare linked requirements more, if needed."""
reqs = [req for req in reqs if req.needs_more_preparation]
def _ensure_download_info(self, reqs: Iterable[InstallRequirement]) -> None:
"""
`pip install --report` extracts the download info from each requirement for its
JSON output, so we need to make sure every requirement has this before finishing
the resolve. But .download_info will only be populated by the point this method
is called for requirements already found in the wheel cache, so we need to
synthesize it for uncached results. Luckily, a DirectUrl can be parsed directly
from a url without any other context. However, this also means the download info
will only contain a hash if the link itself declares the hash.
"""
for req in reqs:
self._populate_download_info(req)
def _force_fully_prepared(
self, reqs: Iterable[InstallRequirement], require_concrete: bool
) -> None:
"""
The legacy resolver seems to prepare requirements differently that can leave
them half-done in certain code paths. I'm not quite sure how it's doing things,
but at least we can do this to make sure they do things right.
"""
for req in reqs:
req.prepared = True
if require_concrete:
assert req.is_concrete
def finalize_linked_requirements(
self,
reqs: Iterable[InstallRequirement],
hydrate_virtual_reqs: bool,
parallel_builds: bool = False,
) -> None:
"""Prepare linked requirements more, if needed.
Neighboring .metadata files as per PEP 658 or lazy wheels via fast-deps will be
preferred to extract metadata from any concrete requirement (one that has been
mapped to a Link) without downloading the underlying wheel or sdist. When ``pip
install --dry-run`` is called, we want to avoid ever downloading the underlying
dist, but we still need to provide all of the results that pip commands expect
from the typical resolve process.
Those expectations vary, but one distinction lies in whether the command needs
an actual physical dist somewhere on the filesystem, or just the metadata about
it from the resolver (as in ``pip install --report``). If the command requires
actual physical filesystem locations for the resolved dists, it must call this
method with ``hydrate_virtual_reqs=True`` to fully download anything
that remains.
"""
if not hydrate_virtual_reqs:
self._ensure_download_info(reqs)
self._force_fully_prepared(reqs, require_concrete=False)
return
partially_downloaded_reqs: List[InstallRequirement] = []
for req in reqs:
if req.is_concrete:
continue
# Determine if any of these requirements were already downloaded.
if self.download_dir is not None and req.link.is_wheel:
hashes = self._get_linked_req_hashes(req)
file_path = _check_download_dir(req.link, self.download_dir, hashes)
# If the file is there, but doesn't match the hash, delete it and print
# a warning. We will be downloading it again via
# partially_downloaded_reqs.
file_path = _check_download_dir(
req.link, self.download_dir, hashes, warn_on_hash_mismatch=True
)
if file_path is not None:
# If the hash does match, then we still need to generate a concrete
# dist, but we don't have to download the wheel again.
self._downloaded[req.link.url] = file_path
req.needs_more_preparation = False
partially_downloaded_reqs.append(req)
# Prepare requirements we found were already downloaded for some
# reason. The other downloads will be completed separately.
partially_downloaded_reqs: List[InstallRequirement] = []
for req in reqs:
if req.needs_more_preparation:
partially_downloaded_reqs.append(req)
else:
self._prepare_linked_requirement(req, parallel_builds)
# TODO: separate this part out from RequirementPreparer when the v1
# resolver can be removed!
self._complete_partial_requirements(
partially_downloaded_reqs,
parallel_builds=parallel_builds,
)
# NB: Must call this method before returning!
self._force_fully_prepared(reqs, require_concrete=True)
def _prepare_linked_requirement(
self, req: InstallRequirement, parallel_builds: bool
@ -612,12 +813,31 @@ class RequirementPreparer:
hashes.check_against_path(file_path)
local_file = File(file_path, content_type=None)
# For use in later processing,
# preserve the file path on the requirement.
if local_file:
req.local_file_path = local_file.path
self._populate_download_info(req)
(builds_metadata, dist) = _get_prepared_distribution(
req,
self.build_tracker,
self.finder,
self.build_isolation,
self.check_build_deps,
)
if builds_metadata and should_cache(req):
self._cache_metadata(req, dist)
return dist
def _populate_download_info(self, req: InstallRequirement) -> None:
# If download_info is set, we got it from the wheel cache.
if req.download_info is None:
# Editables don't go through this function (see
# prepare_editable_requirement).
assert not req.editable
req.download_info = direct_url_from_link(link, req.source_dir)
req.download_info = direct_url_from_link(req.link, req.source_dir)
# Make sure we have a hash in download_info. If we got it as part of the
# URL, it will have been verified and we can rely on it. Otherwise we
# compute it from the downloaded file.
@ -625,30 +845,17 @@ class RequirementPreparer:
if (
isinstance(req.download_info.info, ArchiveInfo)
and not req.download_info.info.hashes
and local_file
and req.local_file_path
):
hash = hash_file(local_file.path)[0].hexdigest()
hash = hash_file(req.local_file_path)[0].hexdigest()
# We populate info.hash for backward compatibility.
# This will automatically populate info.hashes.
req.download_info.info.hash = f"sha256={hash}"
# For use in later processing,
# preserve the file path on the requirement.
if local_file:
req.local_file_path = local_file.path
dist = _get_prepared_distribution(
req,
self.build_tracker,
self.finder,
self.build_isolation,
self.check_build_deps,
)
return dist
def save_linked_requirement(self, req: InstallRequirement) -> None:
assert self.download_dir is not None
assert req.link is not None
assert req.is_concrete
link = req.link
if link.is_vcs or (link.is_existing_dir() and req.editable):
# Make a .zip of the source_dir we already created.
@ -693,7 +900,7 @@ class RequirementPreparer:
assert req.source_dir
req.download_info = direct_url_for_editable(req.unpacked_source_directory)
dist = _get_prepared_distribution(
(_, dist) = _get_prepared_distribution(
req,
self.build_tracker,
self.finder,
@ -703,6 +910,8 @@ class RequirementPreparer:
req.check_if_exists(self.use_user_site)
# This should already have been populated by the preparation of the source dist.
assert req.is_concrete
return dist
def prepare_installed_requirement(
@ -727,4 +936,13 @@ class RequirementPreparer:
"completely repeatable environment, install into an "
"empty virtualenv."
)
return InstalledDistribution(req).get_metadata_distribution()
(_, dist) = _get_prepared_distribution(
req,
self.build_tracker,
self.finder,
self.build_isolation,
self.check_build_deps,
)
assert req.is_concrete
return dist

104
src/pip/_internal/req/req_install.py
View File

@ -23,10 +23,7 @@ from pip._internal.locations import get_scheme
from pip._internal.metadata import (
BaseDistribution,
get_default_environment,
get_directory_distribution,
get_wheel_distribution,
)
from pip._internal.metadata.base import FilesystemWheel
from pip._internal.models.direct_url import DirectUrl
from pip._internal.models.link import Link
from pip._internal.operations.build.metadata import generate_metadata
@ -88,7 +85,7 @@ class InstallRequirement:
permit_editable_wheels: bool = False,
) -> None:
assert req is None or isinstance(req, Requirement), req
self.req = req
self._req = req
self.comes_from = comes_from
self.constraint = constraint
self.editable = editable
@ -150,6 +147,7 @@ class InstallRequirement:
self.hash_options = hash_options if hash_options else {}
self.config_settings = config_settings
# Set to True after successful preparation of this requirement
# TODO: this is only used in the legacy resolver: remove this!
self.prepared = False
# User supplied requirement are explicitly requested for installation
# by the user via CLI arguments or requirements files, as opposed to,
@ -181,12 +179,29 @@ class InstallRequirement:
# but after loading this flag should be treated as read only.
self.use_pep517 = use_pep517
# This requirement needs more preparation before it can be built
self.needs_more_preparation = False
# When a dist is computed for this requirement, cache it here so it's visible
# everywhere within pip and isn't computed more than once. This may be
# a "virtual" dist without a physical location on the filesystem, or
# a "concrete" dist which has been fully downloaded.
self._cached_dist: Optional[BaseDistribution] = None
# Strictly used in testing: allow calling .cache_concrete_dist() twice.
self.allow_concrete_dist_overwrite = False
# This requirement needs to be unpacked before it can be installed.
self._archive_source: Optional[Path] = None
@property
def req(self) -> Optional[Requirement]:
"""Calculate a requirement from the cached dist if necessary."""
if self._req is not None:
return self._req
if self._cached_dist is not None:
name = self._cached_dist.canonical_name
version = str(self._cached_dist.version)
self._req = Requirement(f"{name}=={version}")
return self._req
return None
def __str__(self) -> str:
if self.req:
s = redact_auth_from_requirement(self.req)
@ -234,7 +249,7 @@ class InstallRequirement:
return None
return self.req.name
@functools.lru_cache() # use cached_property in python 3.8+
@functools.lru_cache(maxsize=None) # TODO: use cached_property in python 3.8+
def supports_pyproject_editable(self) -> bool:
if not self.use_pep517:
return False
@ -380,7 +395,7 @@ class InstallRequirement:
def _set_requirement(self) -> None:
"""Set requirement after generating metadata."""
assert self.req is None
assert self._req is None
assert self.metadata is not None
assert self.source_dir is not None
@ -390,7 +405,7 @@ class InstallRequirement:
else:
op = "==="
self.req = Requirement(
self._req = Requirement(
"".join(
[
self.metadata["Name"],
@ -416,7 +431,7 @@ class InstallRequirement:
metadata_name,
self.name,
)
self.req = Requirement(metadata_name)
self._req = Requirement(metadata_name)
def check_if_exists(self, use_user_site: bool) -> None:
"""Find an installed distribution that satisfies or conflicts
@ -552,11 +567,11 @@ class InstallRequirement:
f"Consider using a build backend that supports PEP 660."
)
def prepare_metadata(self) -> None:
def prepare_metadata_directory(self) -> None:
"""Ensure that project metadata is available.
Under PEP 517 and PEP 660, call the backend hook to prepare the metadata.
Under legacy processing, call setup.py egg-info.
Under PEP 517 and PEP 660, call the backend hook to prepare the metadata
directory. Under legacy processing, call setup.py egg-info.
"""
assert self.source_dir, f"No source dir for {self}"
details = self.name or f"from {self.link}"
@ -588,6 +603,8 @@ class InstallRequirement:
details=details,
)
def validate_sdist_metadata(self) -> None:
"""Ensure that we have a dist, and ensure it corresponds to expectations."""
# Act on the newly generated metadata, based on the name and version.
if not self.name:
self._set_requirement()
@ -598,24 +615,59 @@ class InstallRequirement:
@property
def metadata(self) -> Any:
# TODO: use cached_property in python 3.8+
if not hasattr(self, "_metadata"):
self._metadata = self.get_dist().metadata
self._metadata = self.cached_dist.metadata
return self._metadata
def get_dist(self) -> BaseDistribution:
if self.metadata_directory:
return get_directory_distribution(self.metadata_directory)
elif self.local_file_path and self.is_wheel:
assert self.req is not None
return get_wheel_distribution(
FilesystemWheel(self.local_file_path),
canonicalize_name(self.req.name),
@property
def cached_dist(self) -> BaseDistribution:
"""Retrieve the dist resolved from this requirement.
:raises AssertionError: if the resolver has not yet been executed.
"""
if self._cached_dist is None:
raise AssertionError(
f"InstallRequirement {self} has no dist; "
"ensure the resolver has been executed"
)
raise AssertionError(
f"InstallRequirement {self} has no metadata directory and no wheel: "
f"can't make a distribution."
)
return self._cached_dist
def cache_virtual_metadata_only_dist(self, dist: BaseDistribution) -> None:
"""Associate a "virtual" metadata-only dist to this requirement.
This dist cannot be installed, but it can be used to complete the resolve
process.
:raises AssertionError: if a dist has already been associated.
:raises AssertionError: if the provided dist is "concrete", i.e. exists
somewhere on the filesystem.
"""
assert self._cached_dist is None, self
assert not dist.is_concrete, dist
self._cached_dist = dist
def cache_concrete_dist(self, dist: BaseDistribution) -> None:
"""Associate a "concrete" dist to this requirement.
A concrete dist exists somewhere on the filesystem and can be installed.
:raises AssertionError: if a concrete dist has already been associated.
:raises AssertionError: if the provided dist is not concrete.
"""
if self._cached_dist is not None:
# If we set a dist twice for the same requirement, we must be hydrating
# a concrete dist for what was previously virtual. This will occur in the
# case of `install --dry-run` when PEP 658 metadata is available.
if not self.allow_concrete_dist_overwrite:
assert not self._cached_dist.is_concrete
assert dist.is_concrete
self._cached_dist = dist
@property
def is_concrete(self) -> bool:
return self._cached_dist is not None and self._cached_dist.is_concrete
def assert_source_matches_version(self) -> None:
assert self.source_dir, f"No source dir for {self}"

6
src/pip/_internal/req/req_set.py
View File

@ -46,7 +46,7 @@ class RequirementSet:
self.unnamed_requirements.append(install_req)
def add_named_requirement(self, install_req: InstallRequirement) -> None:
assert install_req.name
assert install_req.name, install_req
project_name = canonicalize_name(install_req.name)
self.requirements[project_name] = install_req
@ -86,7 +86,7 @@ class RequirementSet:
def warn_legacy_versions_and_specifiers(self) -> None:
for req in self.requirements_to_install:
version = req.get_dist().version
version = req.cached_dist.version
if isinstance(version, LegacyVersion):
deprecated(
reason=(
@ -101,7 +101,7 @@ class RequirementSet:
issue=12063,
gone_in="24.0",
)
for dep in req.get_dist().iter_dependencies():
for dep in req.cached_dist.iter_dependencies():
if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier):
deprecated(
reason=(

5
src/pip/_internal/resolution/resolvelib/resolver.py
View File

@ -175,11 +175,6 @@ class Resolver(BaseResolver):
req_set.add_named_requirement(ireq)
reqs = req_set.all_requirements
self.factory.preparer.prepare_linked_requirements_more(reqs)
for req in reqs:
req.prepared = True
req.needs_more_preparation = False
return req_set
def get_installation_order(

62
src/pip/_internal/wheel_builder.py
View File

@ -3,7 +3,6 @@
import logging
import os.path
import re
import shutil
from typing import Iterable, List, Optional, Tuple
@ -25,23 +24,12 @@ from pip._internal.utils.setuptools_build import make_setuptools_clean_args
from pip._internal.utils.subprocess import call_subprocess
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.urls import path_to_url
from pip._internal.vcs import vcs
logger = logging.getLogger(__name__)
_egg_info_re = re.compile(r"([a-z0-9_.]+)-([a-z0-9_.!+-]+)", re.IGNORECASE)
BuildResult = Tuple[List[InstallRequirement], List[InstallRequirement]]
def _contains_egg_info(s: str) -> bool:
"""Determine whether the string looks like an egg_info.
:param s: The string to parse. E.g. foo-2.1
"""
return bool(_egg_info_re.search(s))
def _should_build(
req: InstallRequirement,
need_wheel: bool,
@ -87,54 +75,6 @@ def should_build_for_install_command(
return _should_build(req, need_wheel=False)
def _should_cache(
req: InstallRequirement,
) -> Optional[bool]:
"""
Return whether a built InstallRequirement can be stored in the persistent
wheel cache, assuming the wheel cache is available, and _should_build()
has determined a wheel needs to be built.
"""
if req.editable or not req.source_dir:
# never cache editable requirements
return False
if req.link and req.link.is_vcs:
# VCS checkout. Do not cache
# unless it points to an immutable commit hash.
assert not req.editable
assert req.source_dir
vcs_backend = vcs.get_backend_for_scheme(req.link.scheme)
assert vcs_backend
if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir):
return True
return False
assert req.link
base, ext = req.link.splitext()
if _contains_egg_info(base):
return True
# Otherwise, do not cache.
return False
def _get_cache_dir(
req: InstallRequirement,
wheel_cache: WheelCache,
) -> str:
"""Return the persistent or temporary cache directory where the built
wheel need to be stored.
"""
cache_available = bool(wheel_cache.cache_dir)
assert req.link
if cache_available and _should_cache(req):
cache_dir = wheel_cache.get_path_for_link(req.link)
else:
cache_dir = wheel_cache.get_ephem_path_for_link(req.link)
return cache_dir
def _verify_one(req: InstallRequirement, wheel_path: str) -> None:
canonical_name = canonicalize_name(req.name or "")
w = Wheel(os.path.basename(wheel_path))
@ -315,7 +255,7 @@ def build(
build_successes, build_failures = [], []
for req in requirements:
assert req.name
cache_dir = _get_cache_dir(req, wheel_cache)
cache_dir = wheel_cache.resolve_cache_dir(req)
wheel_file = _build_one(
req,
cache_dir,

32
tests/conftest.py
View File

@ -709,6 +709,9 @@ class FakePackage:
requires_dist: Tuple[str, ...] = ()
# This will override the Name specified in the actual dist's METADATA.
metadata_name: Optional[str] = None
# Whether to delete the file this points to, which causes any attempt to fetch this
# package to fail unless it is processed as a metadata-only dist.
delete_linked_file: bool = False
def metadata_filename(self) -> str:
"""This is specified by PEP 658."""
@ -798,6 +801,27 @@ def fake_packages() -> Dict[str, List[FakePackage]]:
("simple==1.0",),
),
],
"complex-dist": [
FakePackage(
"complex-dist",
"0.1",
"complex_dist-0.1-py2.py3-none-any.whl",
MetadataKind.Unhashed,
# Validate that the wheel isn't fetched if metadata is available and
# --dry-run is on, when the metadata presents no hash itself.
delete_linked_file=True,
),
],
"corruptwheel": [
FakePackage(
"corruptwheel",
"1.0",
"corruptwheel-1.0-py2.py3-none-any.whl",
# Validate that the wheel isn't fetched if metadata is available and
# --dry-run is on, when the metadata *does* present a hash.
MetadataKind.Sha256,
),
],
"has-script": [
# Ensure we check PEP 658 metadata hashing errors for wheel files.
FakePackage(
@ -883,10 +907,10 @@ def html_index_for_packages(
f' <a href="{package_link.filename}" {package_link.generate_additional_tag()}>{package_link.filename}</a><br/>' # noqa: E501
)
# (3.2) Copy over the corresponding file in `shared_data.packages`.
shutil.copy(
shared_data.packages / package_link.filename,
pkg_subdir / package_link.filename,
)
cached_file = shared_data.packages / package_link.filename
new_file = pkg_subdir / package_link.filename
if not package_link.delete_linked_file:
shutil.copy(cached_file, new_file)
# (3.3) Write a metadata file, if applicable.
if package_link.metadata != MetadataKind.NoFile:
with open(pkg_subdir / package_link.metadata_filename(), "wb") as f:

8
tests/functional/test_check.py
View File

@ -119,7 +119,7 @@ def test_check_complicated_name_missing(script: PipTestEnvironment) -> None:
# Without dependency
result = script.pip("install", "--no-index", package_a_path, "--no-deps")
assert "Successfully installed package-A-1.0" in result.stdout, str(result)
assert "Successfully installed package-a-1.0" in result.stdout, str(result)
result = script.pip("check", expect_error=True)
expected_lines = ("package-a 1.0 requires dependency-b, which is not installed.",)
@ -142,7 +142,7 @@ def test_check_complicated_name_broken(script: PipTestEnvironment) -> None:
# With broken dependency
result = script.pip("install", "--no-index", package_a_path, "--no-deps")
assert "Successfully installed package-A-1.0" in result.stdout, str(result)
assert "Successfully installed package-a-1.0" in result.stdout, str(result)
result = script.pip(
"install",
@ -175,7 +175,7 @@ def test_check_complicated_name_clean(script: PipTestEnvironment) -> None:
)
result = script.pip("install", "--no-index", package_a_path, "--no-deps")
assert "Successfully installed package-A-1.0" in result.stdout, str(result)
assert "Successfully installed package-a-1.0" in result.stdout, str(result)
result = script.pip(
"install",
@ -203,7 +203,7 @@ def test_check_considers_conditional_reqs(script: PipTestEnvironment) -> None:
)
result = script.pip("install", "--no-index", package_a_path, "--no-deps")
assert "Successfully installed package-A-1.0" in result.stdout, str(result)
assert "Successfully installed package-a-1.0" in result.stdout, str(result)
result = script.pip("check", expect_error=True)
expected_lines = ("package-a 1.0 requires dependency-b, which is not installed.",)

8
tests/functional/test_install.py
View File

@ -2071,7 +2071,7 @@ def test_install_conflict_results_in_warning(
# Install pkgA without its dependency
result1 = script.pip("install", "--no-index", pkgA_path, "--no-deps")
assert "Successfully installed pkgA-1.0" in result1.stdout, str(result1)
assert "Successfully installed pkga-1.0" in result1.stdout, str(result1)
# Then install an incorrect version of the dependency
result2 = script.pip(
@ -2081,7 +2081,7 @@ def test_install_conflict_results_in_warning(
allow_stderr_error=True,
)
assert "pkga 1.0 requires pkgb==1.0" in result2.stderr, str(result2)
assert "Successfully installed pkgB-2.0" in result2.stdout, str(result2)
assert "Successfully installed pkgb-2.0" in result2.stdout, str(result2)
def test_install_conflict_warning_can_be_suppressed(
@ -2101,11 +2101,11 @@ def test_install_conflict_warning_can_be_suppressed(
# Install pkgA without its dependency
result1 = script.pip("install", "--no-index", pkgA_path, "--no-deps")
assert "Successfully installed pkgA-1.0" in result1.stdout, str(result1)
assert "Successfully installed pkga-1.0" in result1.stdout, str(result1)
# Then install an incorrect version of the dependency; suppressing warning
result2 = script.pip("install", "--no-index", pkgB_path, "--no-warn-conflicts")
assert "Successfully installed pkgB-2.0" in result2.stdout, str(result2)
assert "Successfully installed pkgb-2.0" in result2.stdout, str(result2)
def test_target_install_ignores_distutils_config_install_prefix(

2
tests/functional/test_install_check.py
View File

@ -28,7 +28,7 @@ def test_check_install_canonicalization(script: PipTestEnvironment) -> None:
# Let's install pkgA without its dependency
result = script.pip("install", "--no-index", pkga_path, "--no-deps")
assert "Successfully installed pkgA-1.0" in result.stdout, str(result)
assert "Successfully installed pkga-1.0" in result.stdout, str(result)
# Install the first missing dependency. Only an error for the
# second dependency should remain.

239
tests/functional/test_install_metadata.py Normal file
View File

@ -0,0 +1,239 @@
import json
import re
from pathlib import Path
from typing import Any, Callable, Dict, Iterator, List, Tuple
import pytest
from pip._vendor.packaging.requirements import Requirement
from pip._internal.models.direct_url import DirectUrl
from pip._internal.utils.urls import path_to_url
from tests.lib import (
PipTestEnvironment,
TestPipResult,
)
@pytest.fixture(scope="function")
def install_with_generated_html_index(
script: PipTestEnvironment,
html_index_for_packages: Path,
tmpdir: Path,
) -> Callable[..., Tuple[TestPipResult, Dict[str, Any]]]:
"""Execute `pip download` against a generated PyPI index."""
output_file = tmpdir / "output_file.json"
def run_for_generated_index(
args: List[str],
*,
dry_run: bool = True,
allow_error: bool = False,
) -> Tuple[TestPipResult, Dict[str, Any]]:
"""
Produce a PyPI directory structure pointing to the specified packages, then
execute `pip install --report ... -i ...` pointing to our generated index.
"""
pip_args = [
"install",
*(("--dry-run",) if dry_run else ()),
"--ignore-installed",
"--report",
str(output_file),
"-i",
path_to_url(str(html_index_for_packages)),
*args,
]
result = script.pip(*pip_args, allow_error=allow_error)
try:
with open(output_file, "rb") as f:
report = json.load(f)
except FileNotFoundError:
if allow_error:
report = {}
else:
raise
return (result, report)
return run_for_generated_index
def iter_dists(report: Dict[str, Any]) -> Iterator[Tuple[Requirement, DirectUrl]]:
"""Parse a (req,url) tuple from each installed dist in the --report json."""
for inst in report["install"]:
metadata = inst["metadata"]
name = metadata["name"]
version = metadata["version"]
req = Requirement(f"{name}=={version}")
direct_url = DirectUrl.from_dict(inst["download_info"])
yield (req, direct_url)
@pytest.mark.parametrize(
"requirement_to_install, expected_outputs",
[
("simple2==1.0", ["simple2==1.0", "simple==1.0"]),
("simple==2.0", ["simple==2.0"]),
(
"colander",
["colander==0.9.9", "translationstring==1.1"],
),
(
"compilewheel",
["compilewheel==1.0", "simple==1.0"],
),
],
)
def test_install_with_metadata(
install_with_generated_html_index: Callable[
..., Tuple[TestPipResult, Dict[str, Any]]
],
requirement_to_install: str,
expected_outputs: List[str],
) -> None:
"""Verify that if a data-dist-info-metadata attribute is present, then it is used
instead of the actual dist's METADATA."""
_, report = install_with_generated_html_index(
[requirement_to_install],
)
installed = sorted(str(r) for r, _ in iter_dists(report))
assert installed == expected_outputs
@pytest.mark.parametrize(
"requirement_to_install, real_hash",
[
(
"simple==3.0",
"95e0f200b6302989bcf2cead9465cf229168295ea330ca30d1ffeab5c0fed996",
),
(
"has-script",
"16ba92d7f6f992f6de5ecb7d58c914675cf21f57f8e674fb29dcb4f4c9507e5b",
),
],
)
def test_incorrect_metadata_hash(
install_with_generated_html_index: Callable[
..., Tuple[TestPipResult, Dict[str, Any]]
],
requirement_to_install: str,
real_hash: str,
) -> None:
"""Verify that if a hash for data-dist-info-metadata is provided, it must match the
actual hash of the metadata file."""
result, _ = install_with_generated_html_index(
[requirement_to_install],
allow_error=True,
)
assert result.returncode != 0
expected_msg = f"""\
Expected sha256 WRONG-HASH
Got {real_hash}"""
assert expected_msg in result.stderr
@pytest.mark.parametrize(
"requirement_to_install, expected_url",
[
("simple2==2.0", "simple2-2.0.tar.gz.metadata"),
("priority", "priority-1.0-py2.py3-none-any.whl.metadata"),
],
)
def test_metadata_not_found(
install_with_generated_html_index: Callable[
..., Tuple[TestPipResult, Dict[str, Any]]
],
requirement_to_install: str,
expected_url: str,
) -> None:
"""Verify that if a data-dist-info-metadata attribute is provided, that pip will
fetch the .metadata file at the location specified by PEP 658, and error
if unavailable."""
result, _ = install_with_generated_html_index(
[requirement_to_install],
allow_error=True,
)
assert result.returncode != 0
expected_re = re.escape(expected_url)
pattern = re.compile(
f"ERROR: 404 Client Error: FileNotFoundError for url:.*{expected_re}"
)
assert pattern.search(result.stderr), (pattern, result.stderr)
def test_produces_error_for_mismatched_package_name_in_metadata(
install_with_generated_html_index: Callable[
..., Tuple[TestPipResult, Dict[str, Any]]
],
) -> None:
"""Verify that the package name from the metadata matches the requested package."""
result, _ = install_with_generated_html_index(
["simple2==3.0"],
allow_error=True,
)
assert result.returncode != 0
assert (
"simple2-3.0.tar.gz has inconsistent Name: expected 'simple2', but metadata "
"has 'not-simple2'"
) in result.stdout
@pytest.mark.parametrize(
"requirement",
(
"requires-simple-extra==0.1",
"REQUIRES_SIMPLE-EXTRA==0.1",
"REQUIRES....simple-_-EXTRA==0.1",
),
)
def test_canonicalizes_package_name_before_verifying_metadata(
install_with_generated_html_index: Callable[
..., Tuple[TestPipResult, Dict[str, Any]]
],
requirement: str,
) -> None:
"""Verify that the package name from the command line and the package's
METADATA are both canonicalized before comparison, while the name from the METADATA
is always used verbatim to represent the installed candidate in --report.
Regression test for https://github.com/pypa/pip/issues/12038
"""
_, report = install_with_generated_html_index(
[requirement],
)
reqs = [str(r) for r, _ in iter_dists(report)]
assert reqs == ["Requires_Simple.Extra==0.1"]
@pytest.mark.parametrize(
"requirement,err_string",
(
# It's important that we verify pip won't even attempt to fetch the file, so we
# construct an input that will cause it to error if it tries at all.
(
"complex-dist==0.1",
"Could not install packages due to an OSError: [Errno 2] No such file or directory", # noqa: E501
),
("corruptwheel==1.0", ".whl is invalid."),
),
)
def test_dry_run_avoids_downloading_metadata_only_dists(
install_with_generated_html_index: Callable[
..., Tuple[TestPipResult, Dict[str, Any]]
],
requirement: str,
err_string: str,
) -> None:
"""Verify that the underlying dist files are not downloaded at all when
`install --dry-run` is used to resolve dists with PEP 658 metadata."""
_, report = install_with_generated_html_index(
[requirement],
)
assert [requirement] == [str(r) for r, _ in iter_dists(report)]
result, _ = install_with_generated_html_index(
[requirement],
dry_run=False,
allow_error=True,
)
assert result.returncode != 0
assert err_string in result.stderr

2
tests/functional/test_install_reqs.py
View File

@ -620,7 +620,7 @@ def test_install_distribution_full_union(
result = script.pip_install_local(
to_install, f"{to_install}[bar]", f"{to_install}[baz]"
)
assert "Building wheel for LocalExtras" in result.stdout
assert "Building wheel for localextras" in result.stdout
result.did_create(script.site_packages / "simple")
result.did_create(script.site_packages / "singlemodule.py")

2
tests/functional/test_wheel.py
View File

@ -284,7 +284,7 @@ def test_wheel_package_with_latin1_setup(
pkg_to_wheel = data.packages.joinpath("SetupPyLatin1")
result = script.pip("wheel", pkg_to_wheel)
assert "Successfully built SetupPyUTF8" in result.stdout
assert "Successfully built setuppyutf8" in result.stdout
def test_pip_wheel_with_pep518_build_reqs(

1
tests/unit/metadata/test_metadata_pkg_resources.py
View File

@ -102,6 +102,7 @@ def test_wheel_metadata_works() -> None:
metadata=InMemoryMetadata({"METADATA": metadata.as_bytes()}, "<in-memory>"),
project_name=name,
),
concrete=False,
)
assert name == dist.canonical_name == dist.raw_name

21
tests/unit/test_cache.py
View File

@ -1,13 +1,32 @@
import os
from pathlib import Path
import pytest
from pip._vendor.packaging.tags import Tag, interpreter_name, interpreter_version
from pip._internal.cache import WheelCache, _hash_dict
from pip._internal.cache import WheelCache, _contains_egg_info, _hash_dict
from pip._internal.models.link import Link
from pip._internal.utils.misc import ensure_dir
@pytest.mark.parametrize(
"s, expected",
[
# Trivial.
("pip-18.0", True),
# Ambiguous.
("foo-2-2", True),
("im-valid", True),
# Invalid.
("invalid", False),
("im_invalid", False),
],
)
def test_contains_egg_info(s: str, expected: bool) -> None:
result = _contains_egg_info(s)
assert result == expected
def test_falsey_path_none() -> None:
wc = WheelCache("")
assert wc.cache_dir is None

2
tests/unit/test_req.py
View File

@ -154,6 +154,7 @@ class TestRequirementSet:
os.fspath(data.packages.joinpath("LocalEnvironMarker")),
)
req.user_supplied = True
req.allow_concrete_dist_overwrite = True
reqset.add_unnamed_requirement(req)
finder = make_test_finder(find_links=[data.find_links])
with self._basic_resolver(finder) as resolver:
@ -503,6 +504,7 @@ class TestRequirementSet:
with self._basic_resolver(finder) as resolver:
ireq_url = data.packages.joinpath("FSPkg").as_uri()
ireq = get_processed_req_from_line(f"-e {ireq_url}#egg=FSPkg")
ireq.allow_concrete_dist_overwrite = True
reqset = resolver.resolve([ireq], True)
assert len(reqset.all_requirements) == 1
req = reqset.all_requirements[0]

26
tests/unit/test_wheel_builder.py
View File

@ -5,7 +5,7 @@ from typing import Optional, cast
import pytest
from pip._internal import wheel_builder
from pip._internal import cache, wheel_builder
from pip._internal.models.link import Link
from pip._internal.operations.build.wheel_legacy import format_command_result
from pip._internal.req.req_install import InstallRequirement
@ -13,24 +13,6 @@ from pip._internal.vcs.git import Git
from tests.lib import _create_test_package
@pytest.mark.parametrize(
"s, expected",
[
# Trivial.
("pip-18.0", True),
# Ambiguous.
("foo-2-2", True),
("im-valid", True),
# Invalid.
("invalid", False),
("im_invalid", False),
],
)
def test_contains_egg_info(s: str, expected: bool) -> None:
result = wheel_builder._contains_egg_info(s)
assert result == expected
class ReqMock:
def __init__(
self,
@ -128,7 +110,7 @@ def test_should_build_for_wheel_command(req: ReqMock, expected: bool) -> None:
],
)
def test_should_cache(req: ReqMock, expected: bool) -> None:
assert wheel_builder._should_cache(cast(InstallRequirement, req)) is expected
assert cache.should_cache(cast(InstallRequirement, req)) is expected
def test_should_cache_git_sha(tmpdir: Path) -> None:
@ -138,12 +120,12 @@ def test_should_cache_git_sha(tmpdir: Path) -> None:
# a link referencing a sha should be cached
url = "git+https://g.c/o/r@" + commit + "#egg=mypkg"
req = ReqMock(link=Link(url), source_dir=repo_path)
assert wheel_builder._should_cache(cast(InstallRequirement, req))
assert cache.should_cache(cast(InstallRequirement, req))
# a link not referencing a sha should not be cached
url = "git+https://g.c/o/r@master#egg=mypkg"
req = ReqMock(link=Link(url), source_dir=repo_path)
assert not wheel_builder._should_cache(cast(InstallRequirement, req))
assert not cache.should_cache(cast(InstallRequirement, req))
def test_format_command_result__INFO(caplog: pytest.LogCaptureFixture) -> None: