mirror of https://github.com/pypa/pip
use .metadata distribution info when possible
When performing `install --dry-run` and PEP 658 .metadata files are available to guide the resolve, do not download the associated wheels. Rather use the distribution information directly from the .metadata files when reporting the results on the CLI and in the --report file. - describe the new --dry-run behavior - finalize linked requirements immediately after resolve - introduce is_concrete - funnel InstalledDistribution through _get_prepared_distribution() too
This commit is contained in:
parent
7419f08fe6
commit
eb096b126e
|
@ -0,0 +1 @@
|
|||
Avoid downloading any dists in ``install --dry-run`` if PEP 658 ``.metadata`` files or lazy wheels are available.
|
|
@ -130,6 +130,9 @@ class DownloadCommand(RequirementCommand):
|
|||
self.trace_basic_info(finder)
|
||||
|
||||
requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
|
||||
preparer.finalize_linked_requirements(
|
||||
requirement_set.requirements.values(), hydrate_virtual_reqs=True
|
||||
)
|
||||
|
||||
downloaded: List[str] = []
|
||||
for req in requirement_set.requirements.values():
|
||||
|
@ -138,7 +141,6 @@ class DownloadCommand(RequirementCommand):
|
|||
preparer.save_linked_requirement(req)
|
||||
downloaded.append(req.name)
|
||||
|
||||
preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
|
||||
requirement_set.warn_legacy_versions_and_specifiers()
|
||||
|
||||
if downloaded:
|
||||
|
|
|
@ -84,7 +84,8 @@ class InstallCommand(RequirementCommand):
|
|||
help=(
|
||||
"Don't actually install anything, just print what would be. "
|
||||
"Can be used in combination with --ignore-installed "
|
||||
"to 'resolve' the requirements."
|
||||
"to 'resolve' the requirements. If PEP 658 or fast-deps metadata is "
|
||||
"available, --dry-run also avoids downloading the dependency at all."
|
||||
),
|
||||
)
|
||||
self.cmd_opts.add_option(
|
||||
|
@ -377,6 +378,10 @@ class InstallCommand(RequirementCommand):
|
|||
requirement_set = resolver.resolve(
|
||||
reqs, check_supported_wheels=not options.target_dir
|
||||
)
|
||||
preparer.finalize_linked_requirements(
|
||||
requirement_set.requirements.values(),
|
||||
hydrate_virtual_reqs=not options.dry_run,
|
||||
)
|
||||
|
||||
if options.json_report_file:
|
||||
report = InstallationReport(requirement_set.requirements_to_install)
|
||||
|
|
|
@ -145,6 +145,9 @@ class WheelCommand(RequirementCommand):
|
|||
self.trace_basic_info(finder)
|
||||
|
||||
requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
|
||||
preparer.finalize_linked_requirements(
|
||||
requirement_set.requirements.values(), hydrate_virtual_reqs=True
|
||||
)
|
||||
|
||||
reqs_to_build: List[InstallRequirement] = []
|
||||
for req in requirement_set.requirements.values():
|
||||
|
@ -153,7 +156,6 @@ class WheelCommand(RequirementCommand):
|
|||
elif should_build_for_wheel_command(req):
|
||||
reqs_to_build.append(req)
|
||||
|
||||
preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
|
||||
requirement_set.warn_legacy_versions_and_specifiers()
|
||||
|
||||
# build wheels
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
from pip._internal.distributions.base import AbstractDistribution
|
||||
from pip._internal.distributions.installed import InstalledDistribution
|
||||
from pip._internal.distributions.sdist import SourceDistribution
|
||||
from pip._internal.distributions.wheel import WheelDistribution
|
||||
from pip._internal.req.req_install import InstallRequirement
|
||||
|
@ -8,6 +9,10 @@ def make_distribution_for_install_requirement(
|
|||
install_req: InstallRequirement,
|
||||
) -> AbstractDistribution:
|
||||
"""Returns a Distribution for the given InstallRequirement"""
|
||||
# Only pre-installed requirements will have a .satisfied_by dist.
|
||||
if install_req.satisfied_by:
|
||||
return InstalledDistribution(install_req)
|
||||
|
||||
# Editable requirements will always be source distributions. They use the
|
||||
# legacy logic until we create a modern standard for them.
|
||||
if install_req.editable:
|
||||
|
|
|
@ -35,11 +35,17 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
|
|||
|
||||
If None, then this dist has no work to do in the build tracker, and
|
||||
``.prepare_distribution_metadata()`` will not be called."""
|
||||
raise NotImplementedError()
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_metadata_distribution(self) -> BaseDistribution:
|
||||
raise NotImplementedError()
|
||||
"""Generate a concrete ``BaseDistribution`` instance for this artifact.
|
||||
|
||||
The implementation should also cache the result with
|
||||
``self.req.cache_concrete_dist()`` so the distribution is available to other
|
||||
users of the ``InstallRequirement``. This method is not called within the build
|
||||
tracker context, so it should not identify any new setup requirements."""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
def prepare_distribution_metadata(
|
||||
|
@ -48,4 +54,11 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
|
|||
build_isolation: bool,
|
||||
check_build_deps: bool,
|
||||
) -> None:
|
||||
raise NotImplementedError()
|
||||
"""Generate the information necessary to extract metadata from the artifact.
|
||||
|
||||
This method will be executed within the context of ``BuildTracker#track()``, so
|
||||
it needs to fully identify any seutp requirements so they can be added to the
|
||||
same active set of tracked builds, while ``.get_metadata_distribution()`` takes
|
||||
care of generating and caching the ``BaseDistribution`` to expose to the rest of
|
||||
the resolve."""
|
||||
...
|
||||
|
|
|
@ -17,8 +17,10 @@ class InstalledDistribution(AbstractDistribution):
|
|||
return None
|
||||
|
||||
def get_metadata_distribution(self) -> BaseDistribution:
|
||||
assert self.req.satisfied_by is not None, "not actually installed"
|
||||
return self.req.satisfied_by
|
||||
dist = self.req.satisfied_by
|
||||
assert dist is not None, "not actually installed"
|
||||
self.req.cache_concrete_dist(dist)
|
||||
return dist
|
||||
|
||||
def prepare_distribution_metadata(
|
||||
self,
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
import logging
|
||||
from typing import Iterable, Optional, Set, Tuple
|
||||
from typing import Iterable, Set, Tuple
|
||||
|
||||
from pip._internal.build_env import BuildEnvironment
|
||||
from pip._internal.distributions.base import AbstractDistribution
|
||||
from pip._internal.exceptions import InstallationError
|
||||
from pip._internal.index.package_finder import PackageFinder
|
||||
from pip._internal.metadata import BaseDistribution
|
||||
from pip._internal.metadata import BaseDistribution, get_directory_distribution
|
||||
from pip._internal.utils.subprocess import runner_with_spinner_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -19,13 +19,19 @@ class SourceDistribution(AbstractDistribution):
|
|||
"""
|
||||
|
||||
@property
|
||||
def build_tracker_id(self) -> Optional[str]:
|
||||
def build_tracker_id(self) -> str:
|
||||
"""Identify this requirement uniquely by its link."""
|
||||
assert self.req.link
|
||||
return self.req.link.url_without_fragment
|
||||
|
||||
def get_metadata_distribution(self) -> BaseDistribution:
|
||||
return self.req.get_dist()
|
||||
assert (
|
||||
self.req.metadata_directory
|
||||
), "Set as part of .prepare_distribution_metadata()"
|
||||
dist = get_directory_distribution(self.req.metadata_directory)
|
||||
self.req.cache_concrete_dist(dist)
|
||||
self.req.validate_sdist_metadata()
|
||||
return dist
|
||||
|
||||
def prepare_distribution_metadata(
|
||||
self,
|
||||
|
@ -64,7 +70,11 @@ class SourceDistribution(AbstractDistribution):
|
|||
self._raise_conflicts("the backend dependencies", conflicting)
|
||||
if missing:
|
||||
self._raise_missing_reqs(missing)
|
||||
self.req.prepare_metadata()
|
||||
|
||||
# NB: we must still call .cache_concrete_dist() and .validate_sdist_metadata()
|
||||
# before the InstallRequirement itself has been updated with the metadata from
|
||||
# this directory!
|
||||
self.req.prepare_metadata_directory()
|
||||
|
||||
def _prepare_build_backend(self, finder: PackageFinder) -> None:
|
||||
# Isolate in a BuildEnvironment and install the build-time
|
||||
|
|
|
@ -29,7 +29,9 @@ class WheelDistribution(AbstractDistribution):
|
|||
assert self.req.local_file_path, "Set as part of preparation during download"
|
||||
assert self.req.name, "Wheels are never unnamed"
|
||||
wheel = FilesystemWheel(self.req.local_file_path)
|
||||
return get_wheel_distribution(wheel, canonicalize_name(self.req.name))
|
||||
dist = get_wheel_distribution(wheel, canonicalize_name(self.req.name))
|
||||
self.req.cache_concrete_dist(dist)
|
||||
return dist
|
||||
|
||||
def prepare_distribution_metadata(
|
||||
self,
|
||||
|
|
|
@ -104,6 +104,15 @@ class RequiresEntry(NamedTuple):
|
|||
|
||||
|
||||
class BaseDistribution(Protocol):
|
||||
@property
|
||||
def is_concrete(self) -> bool:
|
||||
"""Whether the distribution really exists somewhere on disk.
|
||||
|
||||
If this is false, it has been synthesized from metadata, e.g. via
|
||||
``.from_metadata_file_contents()``, or ``.from_wheel()`` against
|
||||
a ``MemoryWheel``."""
|
||||
raise NotImplementedError()
|
||||
|
||||
@classmethod
|
||||
def from_directory(cls, directory: str) -> "BaseDistribution":
|
||||
"""Load the distribution from a metadata directory.
|
||||
|
@ -681,6 +690,10 @@ class BaseEnvironment:
|
|||
class Wheel(Protocol):
|
||||
location: str
|
||||
|
||||
@property
|
||||
def is_concrete(self) -> bool:
|
||||
raise NotImplementedError()
|
||||
|
||||
def as_zipfile(self) -> zipfile.ZipFile:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
@ -689,6 +702,10 @@ class FilesystemWheel(Wheel):
|
|||
def __init__(self, location: str) -> None:
|
||||
self.location = location
|
||||
|
||||
@property
|
||||
def is_concrete(self) -> bool:
|
||||
return True
|
||||
|
||||
def as_zipfile(self) -> zipfile.ZipFile:
|
||||
return zipfile.ZipFile(self.location, allowZip64=True)
|
||||
|
||||
|
@ -698,5 +715,9 @@ class MemoryWheel(Wheel):
|
|||
self.location = location
|
||||
self.stream = stream
|
||||
|
||||
@property
|
||||
def is_concrete(self) -> bool:
|
||||
return False
|
||||
|
||||
def as_zipfile(self) -> zipfile.ZipFile:
|
||||
return zipfile.ZipFile(self.stream, allowZip64=True)
|
||||
|
|
|
@ -98,16 +98,22 @@ class Distribution(BaseDistribution):
|
|||
dist: importlib.metadata.Distribution,
|
||||
info_location: Optional[BasePath],
|
||||
installed_location: Optional[BasePath],
|
||||
concrete: bool,
|
||||
) -> None:
|
||||
self._dist = dist
|
||||
self._info_location = info_location
|
||||
self._installed_location = installed_location
|
||||
self._concrete = concrete
|
||||
|
||||
@property
|
||||
def is_concrete(self) -> bool:
|
||||
return self._concrete
|
||||
|
||||
@classmethod
|
||||
def from_directory(cls, directory: str) -> BaseDistribution:
|
||||
info_location = pathlib.Path(directory)
|
||||
dist = importlib.metadata.Distribution.at(info_location)
|
||||
return cls(dist, info_location, info_location.parent)
|
||||
return cls(dist, info_location, info_location.parent, concrete=True)
|
||||
|
||||
@classmethod
|
||||
def from_metadata_file_contents(
|
||||
|
@ -124,7 +130,7 @@ class Distribution(BaseDistribution):
|
|||
metadata_path.write_bytes(metadata_contents)
|
||||
# Construct dist pointing to the newly created directory.
|
||||
dist = importlib.metadata.Distribution.at(metadata_path.parent)
|
||||
return cls(dist, metadata_path.parent, None)
|
||||
return cls(dist, metadata_path.parent, None, concrete=False)
|
||||
|
||||
@classmethod
|
||||
def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
|
||||
|
@ -135,7 +141,12 @@ class Distribution(BaseDistribution):
|
|||
raise InvalidWheel(wheel.location, name) from e
|
||||
except UnsupportedWheel as e:
|
||||
raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
|
||||
return cls(dist, dist.info_location, pathlib.PurePosixPath(wheel.location))
|
||||
return cls(
|
||||
dist,
|
||||
dist.info_location,
|
||||
pathlib.PurePosixPath(wheel.location),
|
||||
concrete=wheel.is_concrete,
|
||||
)
|
||||
|
||||
@property
|
||||
def location(self) -> Optional[str]:
|
||||
|
|
|
@ -81,7 +81,7 @@ class _DistributionFinder:
|
|||
installed_location: Optional[BasePath] = None
|
||||
else:
|
||||
installed_location = info_location.parent
|
||||
yield Distribution(dist, info_location, installed_location)
|
||||
yield Distribution(dist, info_location, installed_location, concrete=True)
|
||||
|
||||
def find_linked(self, location: str) -> Iterator[BaseDistribution]:
|
||||
"""Read location in egg-link files and return distributions in there.
|
||||
|
@ -105,7 +105,7 @@ class _DistributionFinder:
|
|||
continue
|
||||
target_location = str(path.joinpath(target_rel))
|
||||
for dist, info_location in self._find_impl(target_location):
|
||||
yield Distribution(dist, info_location, path)
|
||||
yield Distribution(dist, info_location, path, concrete=True)
|
||||
|
||||
def _find_eggs_in_dir(self, location: str) -> Iterator[BaseDistribution]:
|
||||
from pip._vendor.pkg_resources import find_distributions
|
||||
|
@ -117,7 +117,7 @@ class _DistributionFinder:
|
|||
if not entry.name.endswith(".egg"):
|
||||
continue
|
||||
for dist in find_distributions(entry.path):
|
||||
yield legacy.Distribution(dist)
|
||||
yield legacy.Distribution(dist, concrete=True)
|
||||
|
||||
def _find_eggs_in_zip(self, location: str) -> Iterator[BaseDistribution]:
|
||||
from pip._vendor.pkg_resources import find_eggs_in_zip
|
||||
|
@ -129,7 +129,7 @@ class _DistributionFinder:
|
|||
except zipimport.ZipImportError:
|
||||
return
|
||||
for dist in find_eggs_in_zip(importer, location):
|
||||
yield legacy.Distribution(dist)
|
||||
yield legacy.Distribution(dist, concrete=True)
|
||||
|
||||
def find_eggs(self, location: str) -> Iterator[BaseDistribution]:
|
||||
"""Find eggs in a location.
|
||||
|
|
|
@ -73,8 +73,13 @@ class InMemoryMetadata:
|
|||
|
||||
|
||||
class Distribution(BaseDistribution):
|
||||
def __init__(self, dist: pkg_resources.Distribution) -> None:
|
||||
def __init__(self, dist: pkg_resources.Distribution, concrete: bool) -> None:
|
||||
self._dist = dist
|
||||
self._concrete = concrete
|
||||
|
||||
@property
|
||||
def is_concrete(self) -> bool:
|
||||
return self._concrete
|
||||
|
||||
@classmethod
|
||||
def from_directory(cls, directory: str) -> BaseDistribution:
|
||||
|
@ -94,7 +99,7 @@ class Distribution(BaseDistribution):
|
|||
dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
|
||||
|
||||
dist = dist_cls(base_dir, project_name=dist_name, metadata=metadata)
|
||||
return cls(dist)
|
||||
return cls(dist, concrete=True)
|
||||
|
||||
@classmethod
|
||||
def from_metadata_file_contents(
|
||||
|
@ -111,7 +116,7 @@ class Distribution(BaseDistribution):
|
|||
metadata=InMemoryMetadata(metadata_dict, filename),
|
||||
project_name=project_name,
|
||||
)
|
||||
return cls(dist)
|
||||
return cls(dist, concrete=False)
|
||||
|
||||
@classmethod
|
||||
def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
|
||||
|
@ -132,7 +137,7 @@ class Distribution(BaseDistribution):
|
|||
metadata=InMemoryMetadata(metadata_dict, wheel.location),
|
||||
project_name=name,
|
||||
)
|
||||
return cls(dist)
|
||||
return cls(dist, concrete=wheel.is_concrete)
|
||||
|
||||
@property
|
||||
def location(self) -> Optional[str]:
|
||||
|
@ -241,7 +246,7 @@ class Environment(BaseEnvironment):
|
|||
|
||||
def _iter_distributions(self) -> Iterator[BaseDistribution]:
|
||||
for dist in self._ws:
|
||||
yield Distribution(dist)
|
||||
yield Distribution(dist, concrete=True)
|
||||
|
||||
def _search_distribution(self, name: str) -> Optional[BaseDistribution]:
|
||||
"""Find a distribution matching the ``name`` in the environment.
|
||||
|
|
|
@ -32,7 +32,7 @@ class InstallationReport:
|
|||
"requested": ireq.user_supplied,
|
||||
# PEP 566 json encoding for metadata
|
||||
# https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
|
||||
"metadata": ireq.get_dist().metadata_dict,
|
||||
"metadata": ireq.cached_dist.metadata_dict,
|
||||
}
|
||||
if ireq.user_supplied and ireq.extras:
|
||||
# For top level requirements, the list of requested extras, if any.
|
||||
|
|
|
@ -9,7 +9,6 @@ from pip._vendor.packaging.specifiers import LegacySpecifier
|
|||
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
|
||||
from pip._vendor.packaging.version import LegacyVersion
|
||||
|
||||
from pip._internal.distributions import make_distribution_for_install_requirement
|
||||
from pip._internal.metadata import get_default_environment
|
||||
from pip._internal.metadata.base import DistributionVersion
|
||||
from pip._internal.req.req_install import InstallRequirement
|
||||
|
@ -127,8 +126,8 @@ def _simulate_installation_of(
|
|||
|
||||
# Modify it as installing requirement_set would (assuming no errors)
|
||||
for inst_req in to_install:
|
||||
abstract_dist = make_distribution_for_install_requirement(inst_req)
|
||||
dist = abstract_dist.get_metadata_distribution()
|
||||
assert inst_req.is_concrete
|
||||
dist = inst_req.cached_dist
|
||||
name = dist.canonical_name
|
||||
package_set[name] = PackageDetails(dist.version, list(dist.iter_dependencies()))
|
||||
|
||||
|
|
|
@ -13,7 +13,6 @@ from typing import Dict, Iterable, List, Optional
|
|||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
|
||||
from pip._internal.distributions import make_distribution_for_install_requirement
|
||||
from pip._internal.distributions.installed import InstalledDistribution
|
||||
from pip._internal.exceptions import (
|
||||
DirectoryUrlHashUnsupported,
|
||||
HashMismatch,
|
||||
|
@ -24,7 +23,10 @@ from pip._internal.exceptions import (
|
|||
VcsHashUnsupported,
|
||||
)
|
||||
from pip._internal.index.package_finder import PackageFinder
|
||||
from pip._internal.metadata import BaseDistribution, get_metadata_distribution
|
||||
from pip._internal.metadata import (
|
||||
BaseDistribution,
|
||||
get_metadata_distribution,
|
||||
)
|
||||
from pip._internal.models.direct_url import ArchiveInfo
|
||||
from pip._internal.models.link import Link
|
||||
from pip._internal.models.wheel import Wheel
|
||||
|
@ -187,6 +189,8 @@ def _check_download_dir(
|
|||
) -> Optional[str]:
|
||||
"""Check download_dir for previously downloaded file with correct hash
|
||||
If a correct file is found return its path else None
|
||||
|
||||
If a file is found at the given path, but with an invalid hash, the file is deleted.
|
||||
"""
|
||||
download_path = os.path.join(download_dir, link.filename)
|
||||
|
||||
|
@ -517,41 +521,92 @@ class RequirementPreparer:
|
|||
# The file is not available, attempt to fetch only metadata
|
||||
metadata_dist = self._fetch_metadata_only(req)
|
||||
if metadata_dist is not None:
|
||||
req.needs_more_preparation = True
|
||||
# These reqs now have the dependency information from the downloaded
|
||||
# metadata, without having downloaded the actual dist at all.
|
||||
req.cache_virtual_metadata_only_dist(metadata_dist)
|
||||
return metadata_dist
|
||||
|
||||
# None of the optimizations worked, fully prepare the requirement
|
||||
return self._prepare_linked_requirement(req, parallel_builds)
|
||||
|
||||
def prepare_linked_requirements_more(
|
||||
self, reqs: Iterable[InstallRequirement], parallel_builds: bool = False
|
||||
) -> None:
|
||||
"""Prepare linked requirements more, if needed."""
|
||||
reqs = [req for req in reqs if req.needs_more_preparation]
|
||||
def _ensure_download_info(self, reqs: Iterable[InstallRequirement]) -> None:
|
||||
"""
|
||||
`pip install --report` extracts the download info from each requirement for its
|
||||
JSON output, so we need to make sure every requirement has this before finishing
|
||||
the resolve. But .download_info will only be populated by the point this method
|
||||
is called for requirements already found in the wheel cache, so we need to
|
||||
synthesize it for uncached results. Luckily, a DirectUrl can be parsed directly
|
||||
from a url without any other context. However, this also means the download info
|
||||
will only contain a hash if the link itself declares the hash.
|
||||
"""
|
||||
for req in reqs:
|
||||
self._populate_download_info(req)
|
||||
|
||||
def _force_fully_prepared(
|
||||
self, reqs: Iterable[InstallRequirement], require_concrete: bool
|
||||
) -> None:
|
||||
"""
|
||||
The legacy resolver seems to prepare requirements differently that can leave
|
||||
them half-done in certain code paths. I'm not quite sure how it's doing things,
|
||||
but at least we can do this to make sure they do things right.
|
||||
"""
|
||||
for req in reqs:
|
||||
req.prepared = True
|
||||
if require_concrete:
|
||||
assert req.is_concrete
|
||||
|
||||
def finalize_linked_requirements(
|
||||
self,
|
||||
reqs: Iterable[InstallRequirement],
|
||||
hydrate_virtual_reqs: bool,
|
||||
parallel_builds: bool = False,
|
||||
) -> None:
|
||||
"""Prepare linked requirements more, if needed.
|
||||
|
||||
Neighboring .metadata files as per PEP 658 or lazy wheels via fast-deps will be
|
||||
preferred to extract metadata from any concrete requirement (one that has been
|
||||
mapped to a Link) without downloading the underlying wheel or sdist. When ``pip
|
||||
install --dry-run`` is called, we want to avoid ever downloading the underlying
|
||||
dist, but we still need to provide all of the results that pip commands expect
|
||||
from the typical resolve process.
|
||||
|
||||
Those expectations vary, but one distinction lies in whether the command needs
|
||||
an actual physical dist somewhere on the filesystem, or just the metadata about
|
||||
it from the resolver (as in ``pip install --report``). If the command requires
|
||||
actual physical filesystem locations for the resolved dists, it must call this
|
||||
method with ``hydrate_virtual_reqs=True`` to fully download anything
|
||||
that remains.
|
||||
"""
|
||||
if not hydrate_virtual_reqs:
|
||||
self._ensure_download_info(reqs)
|
||||
self._force_fully_prepared(reqs, require_concrete=False)
|
||||
return
|
||||
|
||||
partially_downloaded_reqs: List[InstallRequirement] = []
|
||||
for req in reqs:
|
||||
if req.is_concrete:
|
||||
continue
|
||||
# Determine if any of these requirements were already downloaded.
|
||||
if self.download_dir is not None and req.link.is_wheel:
|
||||
hashes = self._get_linked_req_hashes(req)
|
||||
file_path = _check_download_dir(req.link, self.download_dir, hashes)
|
||||
# If the file is there, but doesn't match the hash, delete it and print
|
||||
# a warning. We will be downloading it again via
|
||||
# partially_downloaded_reqs.
|
||||
file_path = _check_download_dir(
|
||||
req.link, self.download_dir, hashes, warn_on_hash_mismatch=True
|
||||
)
|
||||
if file_path is not None:
|
||||
# If the hash does match, then we still need to generate a concrete
|
||||
# dist, but we don't have to download the wheel again.
|
||||
self._downloaded[req.link.url] = file_path
|
||||
req.needs_more_preparation = False
|
||||
partially_downloaded_reqs.append(req)
|
||||
|
||||
# Prepare requirements we found were already downloaded for some
|
||||
# reason. The other downloads will be completed separately.
|
||||
partially_downloaded_reqs: List[InstallRequirement] = []
|
||||
for req in reqs:
|
||||
if req.needs_more_preparation:
|
||||
partially_downloaded_reqs.append(req)
|
||||
else:
|
||||
self._prepare_linked_requirement(req, parallel_builds)
|
||||
|
||||
# TODO: separate this part out from RequirementPreparer when the v1
|
||||
# resolver can be removed!
|
||||
self._complete_partial_requirements(
|
||||
partially_downloaded_reqs,
|
||||
parallel_builds=parallel_builds,
|
||||
)
|
||||
# NB: Must call this method before returning!
|
||||
self._force_fully_prepared(reqs, require_concrete=True)
|
||||
|
||||
def _prepare_linked_requirement(
|
||||
self, req: InstallRequirement, parallel_builds: bool
|
||||
|
@ -611,31 +666,13 @@ class RequirementPreparer:
|
|||
hashes.check_against_path(file_path)
|
||||
local_file = File(file_path, content_type=None)
|
||||
|
||||
# If download_info is set, we got it from the wheel cache.
|
||||
if req.download_info is None:
|
||||
# Editables don't go through this function (see
|
||||
# prepare_editable_requirement).
|
||||
assert not req.editable
|
||||
req.download_info = direct_url_from_link(link, req.source_dir)
|
||||
# Make sure we have a hash in download_info. If we got it as part of the
|
||||
# URL, it will have been verified and we can rely on it. Otherwise we
|
||||
# compute it from the downloaded file.
|
||||
# FIXME: https://github.com/pypa/pip/issues/11943
|
||||
if (
|
||||
isinstance(req.download_info.info, ArchiveInfo)
|
||||
and not req.download_info.info.hashes
|
||||
and local_file
|
||||
):
|
||||
hash = hash_file(local_file.path)[0].hexdigest()
|
||||
# We populate info.hash for backward compatibility.
|
||||
# This will automatically populate info.hashes.
|
||||
req.download_info.info.hash = f"sha256={hash}"
|
||||
|
||||
# For use in later processing,
|
||||
# preserve the file path on the requirement.
|
||||
if local_file:
|
||||
req.local_file_path = local_file.path
|
||||
|
||||
self._populate_download_info(req)
|
||||
|
||||
dist = _get_prepared_distribution(
|
||||
req,
|
||||
self.build_tracker,
|
||||
|
@ -645,9 +682,31 @@ class RequirementPreparer:
|
|||
)
|
||||
return dist
|
||||
|
||||
def _populate_download_info(self, req: InstallRequirement) -> None:
|
||||
# If download_info is set, we got it from the wheel cache.
|
||||
if req.download_info is None:
|
||||
# Editables don't go through this function (see
|
||||
# prepare_editable_requirement).
|
||||
assert not req.editable
|
||||
req.download_info = direct_url_from_link(req.link, req.source_dir)
|
||||
# Make sure we have a hash in download_info. If we got it as part of the
|
||||
# URL, it will have been verified and we can rely on it. Otherwise we
|
||||
# compute it from the downloaded file.
|
||||
# FIXME: https://github.com/pypa/pip/issues/11943
|
||||
if (
|
||||
isinstance(req.download_info.info, ArchiveInfo)
|
||||
and not req.download_info.info.hashes
|
||||
and req.local_file_path
|
||||
):
|
||||
hash = hash_file(req.local_file_path)[0].hexdigest()
|
||||
# We populate info.hash for backward compatibility.
|
||||
# This will automatically populate info.hashes.
|
||||
req.download_info.info.hash = f"sha256={hash}"
|
||||
|
||||
def save_linked_requirement(self, req: InstallRequirement) -> None:
|
||||
assert self.download_dir is not None
|
||||
assert req.link is not None
|
||||
assert req.is_concrete
|
||||
link = req.link
|
||||
if link.is_vcs or (link.is_existing_dir() and req.editable):
|
||||
# Make a .zip of the source_dir we already created.
|
||||
|
@ -702,6 +761,8 @@ class RequirementPreparer:
|
|||
|
||||
req.check_if_exists(self.use_user_site)
|
||||
|
||||
# This should already have been populated by the preparation of the source dist.
|
||||
assert req.is_concrete
|
||||
return dist
|
||||
|
||||
def prepare_installed_requirement(
|
||||
|
@ -726,4 +787,13 @@ class RequirementPreparer:
|
|||
"completely repeatable environment, install into an "
|
||||
"empty virtualenv."
|
||||
)
|
||||
return InstalledDistribution(req).get_metadata_distribution()
|
||||
dist = _get_prepared_distribution(
|
||||
req,
|
||||
self.build_tracker,
|
||||
self.finder,
|
||||
self.build_isolation,
|
||||
self.check_build_deps,
|
||||
)
|
||||
|
||||
assert req.is_concrete
|
||||
return dist
|
||||
|
|
|
@ -23,10 +23,7 @@ from pip._internal.locations import get_scheme
|
|||
from pip._internal.metadata import (
|
||||
BaseDistribution,
|
||||
get_default_environment,
|
||||
get_directory_distribution,
|
||||
get_wheel_distribution,
|
||||
)
|
||||
from pip._internal.metadata.base import FilesystemWheel
|
||||
from pip._internal.models.direct_url import DirectUrl
|
||||
from pip._internal.models.link import Link
|
||||
from pip._internal.operations.build.metadata import generate_metadata
|
||||
|
@ -149,6 +146,7 @@ class InstallRequirement:
|
|||
self.hash_options = hash_options if hash_options else {}
|
||||
self.config_settings = config_settings
|
||||
# Set to True after successful preparation of this requirement
|
||||
# TODO: this is only used in the legacy resolver: remove this!
|
||||
self.prepared = False
|
||||
# User supplied requirement are explicitly requested for installation
|
||||
# by the user via CLI arguments or requirements files, as opposed to,
|
||||
|
@ -180,8 +178,13 @@ class InstallRequirement:
|
|||
# but after loading this flag should be treated as read only.
|
||||
self.use_pep517 = use_pep517
|
||||
|
||||
# This requirement needs more preparation before it can be built
|
||||
self.needs_more_preparation = False
|
||||
# When a dist is computed for this requirement, cache it here so it's visible
|
||||
# everywhere within pip and isn't computed more than once. This may be
|
||||
# a "virtual" dist without a physical location on the filesystem, or
|
||||
# a "concrete" dist which has been fully downloaded.
|
||||
self._cached_dist: Optional[BaseDistribution] = None
|
||||
# Strictly used in testing: allow calling .cache_concrete_dist() twice.
|
||||
self.allow_concrete_dist_overwrite = False
|
||||
|
||||
# This requirement needs to be unpacked before it can be installed.
|
||||
self._archive_source: Optional[Path] = None
|
||||
|
@ -233,7 +236,7 @@ class InstallRequirement:
|
|||
return None
|
||||
return self.req.name
|
||||
|
||||
@functools.lru_cache() # use cached_property in python 3.8+
|
||||
@functools.lru_cache(maxsize=None) # TODO: use cached_property in python 3.8+
|
||||
def supports_pyproject_editable(self) -> bool:
|
||||
if not self.use_pep517:
|
||||
return False
|
||||
|
@ -551,11 +554,11 @@ class InstallRequirement:
|
|||
f"Consider using a build backend that supports PEP 660."
|
||||
)
|
||||
|
||||
def prepare_metadata(self) -> None:
|
||||
def prepare_metadata_directory(self) -> None:
|
||||
"""Ensure that project metadata is available.
|
||||
|
||||
Under PEP 517 and PEP 660, call the backend hook to prepare the metadata.
|
||||
Under legacy processing, call setup.py egg-info.
|
||||
Under PEP 517 and PEP 660, call the backend hook to prepare the metadata
|
||||
directory. Under legacy processing, call setup.py egg-info.
|
||||
"""
|
||||
assert self.source_dir, f"No source dir for {self}"
|
||||
details = self.name or f"from {self.link}"
|
||||
|
@ -587,6 +590,8 @@ class InstallRequirement:
|
|||
details=details,
|
||||
)
|
||||
|
||||
def validate_sdist_metadata(self) -> None:
|
||||
"""Ensure that we have a dist, and ensure it corresponds to expectations."""
|
||||
# Act on the newly generated metadata, based on the name and version.
|
||||
if not self.name:
|
||||
self._set_requirement()
|
||||
|
@ -597,24 +602,59 @@ class InstallRequirement:
|
|||
|
||||
@property
|
||||
def metadata(self) -> Any:
|
||||
# TODO: use cached_property in python 3.8+
|
||||
if not hasattr(self, "_metadata"):
|
||||
self._metadata = self.get_dist().metadata
|
||||
self._metadata = self.cached_dist.metadata
|
||||
|
||||
return self._metadata
|
||||
|
||||
def get_dist(self) -> BaseDistribution:
|
||||
if self.metadata_directory:
|
||||
return get_directory_distribution(self.metadata_directory)
|
||||
elif self.local_file_path and self.is_wheel:
|
||||
assert self.req is not None
|
||||
return get_wheel_distribution(
|
||||
FilesystemWheel(self.local_file_path),
|
||||
canonicalize_name(self.req.name),
|
||||
@property
|
||||
def cached_dist(self) -> BaseDistribution:
|
||||
"""Retrieve the dist resolved from this requirement.
|
||||
|
||||
:raises AssertionError: if the resolver has not yet been executed.
|
||||
"""
|
||||
if self._cached_dist is None:
|
||||
raise AssertionError(
|
||||
f"InstallRequirement {self} has no dist; "
|
||||
"ensure the resolver has been executed"
|
||||
)
|
||||
raise AssertionError(
|
||||
f"InstallRequirement {self} has no metadata directory and no wheel: "
|
||||
f"can't make a distribution."
|
||||
)
|
||||
return self._cached_dist
|
||||
|
||||
def cache_virtual_metadata_only_dist(self, dist: BaseDistribution) -> None:
|
||||
"""Associate a "virtual" metadata-only dist to this requirement.
|
||||
|
||||
This dist cannot be installed, but it can be used to complete the resolve
|
||||
process.
|
||||
|
||||
:raises AssertionError: if a dist has already been associated.
|
||||
:raises AssertionError: if the provided dist is "concrete", i.e. exists
|
||||
somewhere on the filesystem.
|
||||
"""
|
||||
assert self._cached_dist is None, self
|
||||
assert not dist.is_concrete, dist
|
||||
self._cached_dist = dist
|
||||
|
||||
def cache_concrete_dist(self, dist: BaseDistribution) -> None:
|
||||
"""Associate a "concrete" dist to this requirement.
|
||||
|
||||
A concrete dist exists somewhere on the filesystem and can be installed.
|
||||
|
||||
:raises AssertionError: if a concrete dist has already been associated.
|
||||
:raises AssertionError: if the provided dist is not concrete.
|
||||
"""
|
||||
if self._cached_dist is not None:
|
||||
# If we set a dist twice for the same requirement, we must be hydrating
|
||||
# a concrete dist for what was previously virtual. This will occur in the
|
||||
# case of `install --dry-run` when PEP 658 metadata is available.
|
||||
if not self.allow_concrete_dist_overwrite:
|
||||
assert not self._cached_dist.is_concrete
|
||||
assert dist.is_concrete
|
||||
self._cached_dist = dist
|
||||
|
||||
@property
|
||||
def is_concrete(self) -> bool:
|
||||
return self._cached_dist is not None and self._cached_dist.is_concrete
|
||||
|
||||
def assert_source_matches_version(self) -> None:
|
||||
assert self.source_dir, f"No source dir for {self}"
|
||||
|
|
|
@ -86,7 +86,7 @@ class RequirementSet:
|
|||
|
||||
def warn_legacy_versions_and_specifiers(self) -> None:
|
||||
for req in self.requirements_to_install:
|
||||
version = req.get_dist().version
|
||||
version = req.cached_dist.version
|
||||
if isinstance(version, LegacyVersion):
|
||||
deprecated(
|
||||
reason=(
|
||||
|
@ -101,7 +101,7 @@ class RequirementSet:
|
|||
issue=12063,
|
||||
gone_in="23.3",
|
||||
)
|
||||
for dep in req.get_dist().iter_dependencies():
|
||||
for dep in req.cached_dist.iter_dependencies():
|
||||
if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier):
|
||||
deprecated(
|
||||
reason=(
|
||||
|
|
|
@ -157,11 +157,6 @@ class Resolver(BaseResolver):
|
|||
|
||||
req_set.add_named_requirement(ireq)
|
||||
|
||||
reqs = req_set.all_requirements
|
||||
self.factory.preparer.prepare_linked_requirements_more(reqs)
|
||||
for req in reqs:
|
||||
req.prepared = True
|
||||
req.needs_more_preparation = False
|
||||
return req_set
|
||||
|
||||
def get_installation_order(
|
||||
|
|
|
@ -102,6 +102,7 @@ def test_wheel_metadata_works() -> None:
|
|||
metadata=InMemoryMetadata({"METADATA": metadata.as_bytes()}, "<in-memory>"),
|
||||
project_name=name,
|
||||
),
|
||||
concrete=False,
|
||||
)
|
||||
|
||||
assert name == dist.canonical_name == dist.raw_name
|
||||
|
|
|
@ -152,6 +152,7 @@ class TestRequirementSet:
|
|||
os.fspath(data.packages.joinpath("LocalEnvironMarker")),
|
||||
)
|
||||
req.user_supplied = True
|
||||
req.allow_concrete_dist_overwrite = True
|
||||
reqset.add_unnamed_requirement(req)
|
||||
finder = make_test_finder(find_links=[data.find_links])
|
||||
with self._basic_resolver(finder) as resolver:
|
||||
|
@ -501,6 +502,7 @@ class TestRequirementSet:
|
|||
with self._basic_resolver(finder) as resolver:
|
||||
ireq_url = data.packages.joinpath("FSPkg").as_uri()
|
||||
ireq = get_processed_req_from_line(f"-e {ireq_url}#egg=FSPkg")
|
||||
ireq.allow_concrete_dist_overwrite = True
|
||||
reqset = resolver.resolve([ireq], True)
|
||||
assert len(reqset.all_requirements) == 1
|
||||
req = reqset.all_requirements[0]
|
||||
|
|
Loading…
Reference in New Issue