Merge eb096b126e into a15dd75d98

When performing `install --dry-run` and PEP 658 .metadata files are
available to guide the resolve, do not download the associated wheels.

Rather use the distribution information directly from the .metadata
files when reporting the results on the CLI and in the --report file.

- describe the new --dry-run behavior
- finalize linked requirements immediately after resolve
- introduce is_concrete
- funnel InstalledDistribution through _get_prepared_distribution() too

docs/html/cli/pip_install.rst

@@ -45,8 +45,8 @@ When looking at the items to be installed, pip checks what type of item
 each is, in the following order:
 
 1. Project or archive URL.
-2. Local directory (which must contain a ``setup.py``, or pip will report
-   an error).
+2. Local directory (which must contain a ``pyproject.toml`` or ``setup.py``,
+   otherwise pip will report an error).
 3. Local file (a sdist or wheel format archive, following the naming
    conventions for those formats).
 4. A requirement, as specified in :pep:`440`.

news/12186.bugfix.rst

@@ -0,0 +1 @@
+Avoid downloading any dists in ``install --dry-run`` if PEP 658 ``.metadata`` files or lazy wheels are available.

news/12417.doc.rst

@@ -0,0 +1 @@
+Fix outdated pip install argument description in documentation.

src/pip/_internal/commands/download.py

@@ -130,6 +130,9 @@ class DownloadCommand(RequirementCommand):
         self.trace_basic_info(finder)
 
         requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+        preparer.finalize_linked_requirements(
+            requirement_set.requirements.values(), hydrate_virtual_reqs=True
+        )
 
         downloaded: List[str] = []
         for req in requirement_set.requirements.values():
@@ -138,7 +141,6 @@ class DownloadCommand(RequirementCommand):
                 preparer.save_linked_requirement(req)
                 downloaded.append(req.name)
 
-        preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
         requirement_set.warn_legacy_versions_and_specifiers()
 
         if downloaded:

src/pip/_internal/commands/install.py

@@ -84,7 +84,8 @@ class InstallCommand(RequirementCommand):
             help=(
                 "Don't actually install anything, just print what would be. "
                 "Can be used in combination with --ignore-installed "
-                "to 'resolve' the requirements."
+                "to 'resolve' the requirements. If PEP 658 or fast-deps metadata is "
+                "available, --dry-run also avoids downloading the dependency at all."
             ),
         )
         self.cmd_opts.add_option(
@@ -377,6 +378,10 @@ class InstallCommand(RequirementCommand):
             requirement_set = resolver.resolve(
                 reqs, check_supported_wheels=not options.target_dir
             )
+            preparer.finalize_linked_requirements(
+                requirement_set.requirements.values(),
+                hydrate_virtual_reqs=not options.dry_run,
+            )
 
             if options.json_report_file:
                 report = InstallationReport(requirement_set.requirements_to_install)

src/pip/_internal/commands/wheel.py

@@ -145,6 +145,9 @@ class WheelCommand(RequirementCommand):
         self.trace_basic_info(finder)
 
         requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+        preparer.finalize_linked_requirements(
+            requirement_set.requirements.values(), hydrate_virtual_reqs=True
+        )
 
         reqs_to_build: List[InstallRequirement] = []
         for req in requirement_set.requirements.values():
@@ -153,7 +156,6 @@ class WheelCommand(RequirementCommand):
             elif should_build_for_wheel_command(req):
                 reqs_to_build.append(req)
 
-        preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
         requirement_set.warn_legacy_versions_and_specifiers()
 
         # build wheels

src/pip/_internal/distributions/__init__.py

@@ -1,4 +1,5 @@
 from pip._internal.distributions.base import AbstractDistribution
+from pip._internal.distributions.installed import InstalledDistribution
 from pip._internal.distributions.sdist import SourceDistribution
 from pip._internal.distributions.wheel import WheelDistribution
 from pip._internal.req.req_install import InstallRequirement
@@ -8,6 +9,10 @@ def make_distribution_for_install_requirement(
     install_req: InstallRequirement,
 ) -> AbstractDistribution:
     """Returns a Distribution for the given InstallRequirement"""
+    # Only pre-installed requirements will have a .satisfied_by dist.
+    if install_req.satisfied_by:
+        return InstalledDistribution(install_req)
+
     # Editable requirements will always be source distributions. They use the
     # legacy logic until we create a modern standard for them.
     if install_req.editable:

src/pip/_internal/distributions/base.py

@@ -35,11 +35,17 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
 
         If None, then this dist has no work to do in the build tracker, and
         ``.prepare_distribution_metadata()`` will not be called."""
-        raise NotImplementedError()
+        ...
 
     @abc.abstractmethod
     def get_metadata_distribution(self) -> BaseDistribution:
-        raise NotImplementedError()
+        """Generate a concrete ``BaseDistribution`` instance for this artifact.
+
+        The implementation should also cache the result with
+        ``self.req.cache_concrete_dist()`` so the distribution is available to other
+        users of the ``InstallRequirement``. This method is not called within the build
+        tracker context, so it should not identify any new setup requirements."""
+        ...
 
     @abc.abstractmethod
     def prepare_distribution_metadata(
@@ -48,4 +54,11 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
         build_isolation: bool,
         check_build_deps: bool,
     ) -> None:
-        raise NotImplementedError()
+        """Generate the information necessary to extract metadata from the artifact.
+
+        This method will be executed within the context of ``BuildTracker#track()``, so
+        it needs to fully identify any seutp requirements so they can be added to the
+        same active set of tracked builds, while ``.get_metadata_distribution()`` takes
+        care of generating and caching the ``BaseDistribution`` to expose to the rest of
+        the resolve."""
+        ...

src/pip/_internal/distributions/installed.py

@@ -17,8 +17,10 @@ class InstalledDistribution(AbstractDistribution):
         return None
 
     def get_metadata_distribution(self) -> BaseDistribution:
-        assert self.req.satisfied_by is not None, "not actually installed"
-        return self.req.satisfied_by
+        dist = self.req.satisfied_by
+        assert dist is not None, "not actually installed"
+        self.req.cache_concrete_dist(dist)
+        return dist
 
     def prepare_distribution_metadata(
         self,

src/pip/_internal/distributions/sdist.py

@@ -1,11 +1,11 @@
 import logging
-from typing import Iterable, Optional, Set, Tuple
+from typing import Iterable, Set, Tuple
 
 from pip._internal.build_env import BuildEnvironment
 from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.exceptions import InstallationError
 from pip._internal.index.package_finder import PackageFinder
-from pip._internal.metadata import BaseDistribution
+from pip._internal.metadata import BaseDistribution, get_directory_distribution
 from pip._internal.utils.subprocess import runner_with_spinner_message
 
 logger = logging.getLogger(__name__)
@@ -19,13 +19,19 @@ class SourceDistribution(AbstractDistribution):
     """
 
     @property
-    def build_tracker_id(self) -> Optional[str]:
+    def build_tracker_id(self) -> str:
         """Identify this requirement uniquely by its link."""
         assert self.req.link
         return self.req.link.url_without_fragment
 
     def get_metadata_distribution(self) -> BaseDistribution:
-        return self.req.get_dist()
+        assert (
+            self.req.metadata_directory
+        ), "Set as part of .prepare_distribution_metadata()"
+        dist = get_directory_distribution(self.req.metadata_directory)
+        self.req.cache_concrete_dist(dist)
+        self.req.validate_sdist_metadata()
+        return dist
 
     def prepare_distribution_metadata(
         self,
@@ -64,7 +70,11 @@ class SourceDistribution(AbstractDistribution):
                 self._raise_conflicts("the backend dependencies", conflicting)
             if missing:
                 self._raise_missing_reqs(missing)
-        self.req.prepare_metadata()
+
+        # NB: we must still call .cache_concrete_dist() and .validate_sdist_metadata()
+        # before the InstallRequirement itself has been updated with the metadata from
+        # this directory!
+        self.req.prepare_metadata_directory()
 
     def _prepare_build_backend(self, finder: PackageFinder) -> None:
         # Isolate in a BuildEnvironment and install the build-time

src/pip/_internal/distributions/wheel.py

@@ -29,7 +29,9 @@ class WheelDistribution(AbstractDistribution):
         assert self.req.local_file_path, "Set as part of preparation during download"
         assert self.req.name, "Wheels are never unnamed"
         wheel = FilesystemWheel(self.req.local_file_path)
-        return get_wheel_distribution(wheel, canonicalize_name(self.req.name))
+        dist = get_wheel_distribution(wheel, canonicalize_name(self.req.name))
+        self.req.cache_concrete_dist(dist)
+        return dist
 
     def prepare_distribution_metadata(
         self,

src/pip/_internal/metadata/base.py

@@ -104,6 +104,15 @@ class RequiresEntry(NamedTuple):
 
 
 class BaseDistribution(Protocol):
+    @property
+    def is_concrete(self) -> bool:
+        """Whether the distribution really exists somewhere on disk.
+
+        If this is false, it has been synthesized from metadata, e.g. via
+        ``.from_metadata_file_contents()``, or ``.from_wheel()`` against
+        a ``MemoryWheel``."""
+        raise NotImplementedError()
+
     @classmethod
     def from_directory(cls, directory: str) -> "BaseDistribution":
         """Load the distribution from a metadata directory.
@@ -681,6 +690,10 @@ class BaseEnvironment:
 class Wheel(Protocol):
     location: str
 
+    @property
+    def is_concrete(self) -> bool:
+        raise NotImplementedError()
+
     def as_zipfile(self) -> zipfile.ZipFile:
         raise NotImplementedError()
 
@@ -689,6 +702,10 @@ class FilesystemWheel(Wheel):
     def __init__(self, location: str) -> None:
         self.location = location
 
+    @property
+    def is_concrete(self) -> bool:
+        return True
+
     def as_zipfile(self) -> zipfile.ZipFile:
         return zipfile.ZipFile(self.location, allowZip64=True)
 
@@ -698,5 +715,9 @@ class MemoryWheel(Wheel):
         self.location = location
         self.stream = stream
 
+    @property
+    def is_concrete(self) -> bool:
+        return False
+
     def as_zipfile(self) -> zipfile.ZipFile:
         return zipfile.ZipFile(self.stream, allowZip64=True)

src/pip/_internal/metadata/importlib/_dists.py

@@ -98,16 +98,22 @@ class Distribution(BaseDistribution):
         dist: importlib.metadata.Distribution,
         info_location: Optional[BasePath],
         installed_location: Optional[BasePath],
+        concrete: bool,
     ) -> None:
         self._dist = dist
         self._info_location = info_location
         self._installed_location = installed_location
+        self._concrete = concrete
+
+    @property
+    def is_concrete(self) -> bool:
+        return self._concrete
 
     @classmethod
     def from_directory(cls, directory: str) -> BaseDistribution:
         info_location = pathlib.Path(directory)
         dist = importlib.metadata.Distribution.at(info_location)
-        return cls(dist, info_location, info_location.parent)
+        return cls(dist, info_location, info_location.parent, concrete=True)
 
     @classmethod
     def from_metadata_file_contents(
@@ -124,7 +130,7 @@ class Distribution(BaseDistribution):
         metadata_path.write_bytes(metadata_contents)
         # Construct dist pointing to the newly created directory.
         dist = importlib.metadata.Distribution.at(metadata_path.parent)
-        return cls(dist, metadata_path.parent, None)
+        return cls(dist, metadata_path.parent, None, concrete=False)
 
     @classmethod
     def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
@@ -135,7 +141,12 @@ class Distribution(BaseDistribution):
             raise InvalidWheel(wheel.location, name) from e
         except UnsupportedWheel as e:
             raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
-        return cls(dist, dist.info_location, pathlib.PurePosixPath(wheel.location))
+        return cls(
+            dist,
+            dist.info_location,
+            pathlib.PurePosixPath(wheel.location),
+            concrete=wheel.is_concrete,
+        )
 
     @property
     def location(self) -> Optional[str]:

src/pip/_internal/metadata/importlib/_envs.py

@@ -81,7 +81,7 @@ class _DistributionFinder:
                 installed_location: Optional[BasePath] = None
             else:
                 installed_location = info_location.parent
-            yield Distribution(dist, info_location, installed_location)
+            yield Distribution(dist, info_location, installed_location, concrete=True)
 
     def find_linked(self, location: str) -> Iterator[BaseDistribution]:
         """Read location in egg-link files and return distributions in there.
@@ -105,7 +105,7 @@ class _DistributionFinder:
                 continue
             target_location = str(path.joinpath(target_rel))
             for dist, info_location in self._find_impl(target_location):
-                yield Distribution(dist, info_location, path)
+                yield Distribution(dist, info_location, path, concrete=True)
 
     def _find_eggs_in_dir(self, location: str) -> Iterator[BaseDistribution]:
         from pip._vendor.pkg_resources import find_distributions
@@ -117,7 +117,7 @@ class _DistributionFinder:
                 if not entry.name.endswith(".egg"):
                     continue
                 for dist in find_distributions(entry.path):
-                    yield legacy.Distribution(dist)
+                    yield legacy.Distribution(dist, concrete=True)
 
     def _find_eggs_in_zip(self, location: str) -> Iterator[BaseDistribution]:
         from pip._vendor.pkg_resources import find_eggs_in_zip
@@ -129,7 +129,7 @@ class _DistributionFinder:
         except zipimport.ZipImportError:
             return
         for dist in find_eggs_in_zip(importer, location):
-            yield legacy.Distribution(dist)
+            yield legacy.Distribution(dist, concrete=True)
 
     def find_eggs(self, location: str) -> Iterator[BaseDistribution]:
         """Find eggs in a location.

src/pip/_internal/metadata/pkg_resources.py

@@ -73,8 +73,13 @@ class InMemoryMetadata:
 
 
 class Distribution(BaseDistribution):
-    def __init__(self, dist: pkg_resources.Distribution) -> None:
+    def __init__(self, dist: pkg_resources.Distribution, concrete: bool) -> None:
         self._dist = dist
+        self._concrete = concrete
+
+    @property
+    def is_concrete(self) -> bool:
+        return self._concrete
 
     @classmethod
     def from_directory(cls, directory: str) -> BaseDistribution:
@@ -94,7 +99,7 @@ class Distribution(BaseDistribution):
             dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
 
         dist = dist_cls(base_dir, project_name=dist_name, metadata=metadata)
-        return cls(dist)
+        return cls(dist, concrete=True)
 
     @classmethod
     def from_metadata_file_contents(
@@ -111,7 +116,7 @@ class Distribution(BaseDistribution):
             metadata=InMemoryMetadata(metadata_dict, filename),
             project_name=project_name,
         )
-        return cls(dist)
+        return cls(dist, concrete=False)
 
     @classmethod
     def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
@@ -132,7 +137,7 @@ class Distribution(BaseDistribution):
             metadata=InMemoryMetadata(metadata_dict, wheel.location),
             project_name=name,
         )
-        return cls(dist)
+        return cls(dist, concrete=wheel.is_concrete)
 
     @property
     def location(self) -> Optional[str]:
@@ -241,7 +246,7 @@ class Environment(BaseEnvironment):
 
     def _iter_distributions(self) -> Iterator[BaseDistribution]:
         for dist in self._ws:
-            yield Distribution(dist)
+            yield Distribution(dist, concrete=True)
 
     def _search_distribution(self, name: str) -> Optional[BaseDistribution]:
         """Find a distribution matching the ``name`` in the environment.

src/pip/_internal/models/installation_report.py

@@ -32,7 +32,7 @@ class InstallationReport:
             "requested": ireq.user_supplied,
             # PEP 566 json encoding for metadata
             # https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
-            "metadata": ireq.get_dist().metadata_dict,
+            "metadata": ireq.cached_dist.metadata_dict,
         }
         if ireq.user_supplied and ireq.extras:
             # For top level requirements, the list of requested extras, if any.

src/pip/_internal/operations/check.py

@@ -9,7 +9,6 @@ from pip._vendor.packaging.specifiers import LegacySpecifier
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 from pip._vendor.packaging.version import LegacyVersion
 
-from pip._internal.distributions import make_distribution_for_install_requirement
 from pip._internal.metadata import get_default_environment
 from pip._internal.metadata.base import DistributionVersion
 from pip._internal.req.req_install import InstallRequirement
@@ -127,8 +126,8 @@ def _simulate_installation_of(
 
     # Modify it as installing requirement_set would (assuming no errors)
     for inst_req in to_install:
-        abstract_dist = make_distribution_for_install_requirement(inst_req)
-        dist = abstract_dist.get_metadata_distribution()
+        assert inst_req.is_concrete
+        dist = inst_req.cached_dist
         name = dist.canonical_name
         package_set[name] = PackageDetails(dist.version, list(dist.iter_dependencies()))
 

src/pip/_internal/operations/prepare.py

@@ -13,7 +13,6 @@ from typing import Dict, Iterable, List, Optional
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.distributions import make_distribution_for_install_requirement
-from pip._internal.distributions.installed import InstalledDistribution
 from pip._internal.exceptions import (
     DirectoryUrlHashUnsupported,
     HashMismatch,
@@ -24,7 +23,10 @@ from pip._internal.exceptions import (
     VcsHashUnsupported,
 )
 from pip._internal.index.package_finder import PackageFinder
-from pip._internal.metadata import BaseDistribution, get_metadata_distribution
+from pip._internal.metadata import (
+    BaseDistribution,
+    get_metadata_distribution,
+)
 from pip._internal.models.direct_url import ArchiveInfo
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
@@ -188,6 +190,8 @@ def _check_download_dir(
 ) -> Optional[str]:
     """Check download_dir for previously downloaded file with correct hash
     If a correct file is found return its path else None
+
+    If a file is found at the given path, but with an invalid hash, the file is deleted.
     """
     download_path = os.path.join(download_dir, link.filename)
 
@@ -518,41 +522,92 @@ class RequirementPreparer:
                 # The file is not available, attempt to fetch only metadata
                 metadata_dist = self._fetch_metadata_only(req)
                 if metadata_dist is not None:
-                    req.needs_more_preparation = True
+                    # These reqs now have the dependency information from the downloaded
+                    # metadata, without having downloaded the actual dist at all.
+                    req.cache_virtual_metadata_only_dist(metadata_dist)
                     return metadata_dist
 
             # None of the optimizations worked, fully prepare the requirement
             return self._prepare_linked_requirement(req, parallel_builds)
 
-    def prepare_linked_requirements_more(
-        self, reqs: Iterable[InstallRequirement], parallel_builds: bool = False
-    ) -> None:
-        """Prepare linked requirements more, if needed."""
-        reqs = [req for req in reqs if req.needs_more_preparation]
+    def _ensure_download_info(self, reqs: Iterable[InstallRequirement]) -> None:
+        """
+        `pip install --report` extracts the download info from each requirement for its
+        JSON output, so we need to make sure every requirement has this before finishing
+        the resolve. But .download_info will only be populated by the point this method
+        is called for requirements already found in the wheel cache, so we need to
+        synthesize it for uncached results. Luckily, a DirectUrl can be parsed directly
+        from a url without any other context. However, this also means the download info
+        will only contain a hash if the link itself declares the hash.
+        """
         for req in reqs:
+            self._populate_download_info(req)
+
+    def _force_fully_prepared(
+        self, reqs: Iterable[InstallRequirement], require_concrete: bool
+    ) -> None:
+        """
+        The legacy resolver seems to prepare requirements differently that can leave
+        them half-done in certain code paths. I'm not quite sure how it's doing things,
+        but at least we can do this to make sure they do things right.
+        """
+        for req in reqs:
+            req.prepared = True
+            if require_concrete:
+                assert req.is_concrete
+
+    def finalize_linked_requirements(
+        self,
+        reqs: Iterable[InstallRequirement],
+        hydrate_virtual_reqs: bool,
+        parallel_builds: bool = False,
+    ) -> None:
+        """Prepare linked requirements more, if needed.
+
+        Neighboring .metadata files as per PEP 658 or lazy wheels via fast-deps will be
+        preferred to extract metadata from any concrete requirement (one that has been
+        mapped to a Link) without downloading the underlying wheel or sdist. When ``pip
+        install --dry-run`` is called, we want to avoid ever downloading the underlying
+        dist, but we still need to provide all of the results that pip commands expect
+        from the typical resolve process.
+
+        Those expectations vary, but one distinction lies in whether the command needs
+        an actual physical dist somewhere on the filesystem, or just the metadata about
+        it from the resolver (as in ``pip install --report``). If the command requires
+        actual physical filesystem locations for the resolved dists, it must call this
+        method with ``hydrate_virtual_reqs=True`` to fully download anything
+        that remains.
+        """
+        if not hydrate_virtual_reqs:
+            self._ensure_download_info(reqs)
+            self._force_fully_prepared(reqs, require_concrete=False)
+            return
+
+        partially_downloaded_reqs: List[InstallRequirement] = []
+        for req in reqs:
+            if req.is_concrete:
+                continue
             # Determine if any of these requirements were already downloaded.
             if self.download_dir is not None and req.link.is_wheel:
                 hashes = self._get_linked_req_hashes(req)
-                file_path = _check_download_dir(req.link, self.download_dir, hashes)
+                # If the file is there, but doesn't match the hash, delete it and print
+                # a warning. We will be downloading it again via
+                # partially_downloaded_reqs.
+                file_path = _check_download_dir(
+                    req.link, self.download_dir, hashes, warn_on_hash_mismatch=True
+                )
                 if file_path is not None:
+                    # If the hash does match, then we still need to generate a concrete
+                    # dist, but we don't have to download the wheel again.
                     self._downloaded[req.link.url] = file_path
-                    req.needs_more_preparation = False
+            partially_downloaded_reqs.append(req)
 
-        # Prepare requirements we found were already downloaded for some
-        # reason. The other downloads will be completed separately.
-        partially_downloaded_reqs: List[InstallRequirement] = []
-        for req in reqs:
-            if req.needs_more_preparation:
-                partially_downloaded_reqs.append(req)
-            else:
-                self._prepare_linked_requirement(req, parallel_builds)
-
-        # TODO: separate this part out from RequirementPreparer when the v1
-        # resolver can be removed!
         self._complete_partial_requirements(
             partially_downloaded_reqs,
             parallel_builds=parallel_builds,
         )
+        # NB: Must call this method before returning!
+        self._force_fully_prepared(reqs, require_concrete=True)
 
     def _prepare_linked_requirement(
         self, req: InstallRequirement, parallel_builds: bool
@@ -612,31 +667,13 @@ class RequirementPreparer:
                 hashes.check_against_path(file_path)
             local_file = File(file_path, content_type=None)
 
-        # If download_info is set, we got it from the wheel cache.
-        if req.download_info is None:
-            # Editables don't go through this function (see
-            # prepare_editable_requirement).
-            assert not req.editable
-            req.download_info = direct_url_from_link(link, req.source_dir)
-            # Make sure we have a hash in download_info. If we got it as part of the
-            # URL, it will have been verified and we can rely on it. Otherwise we
-            # compute it from the downloaded file.
-            # FIXME: https://github.com/pypa/pip/issues/11943
-            if (
-                isinstance(req.download_info.info, ArchiveInfo)
-                and not req.download_info.info.hashes
-                and local_file
-            ):
-                hash = hash_file(local_file.path)[0].hexdigest()
-                # We populate info.hash for backward compatibility.
-                # This will automatically populate info.hashes.
-                req.download_info.info.hash = f"sha256={hash}"
-
         # For use in later processing,
         # preserve the file path on the requirement.
         if local_file:
             req.local_file_path = local_file.path
 
+        self._populate_download_info(req)
+
         dist = _get_prepared_distribution(
             req,
             self.build_tracker,
@@ -646,9 +683,31 @@ class RequirementPreparer:
         )
         return dist
 
+    def _populate_download_info(self, req: InstallRequirement) -> None:
+        # If download_info is set, we got it from the wheel cache.
+        if req.download_info is None:
+            # Editables don't go through this function (see
+            # prepare_editable_requirement).
+            assert not req.editable
+            req.download_info = direct_url_from_link(req.link, req.source_dir)
+            # Make sure we have a hash in download_info. If we got it as part of the
+            # URL, it will have been verified and we can rely on it. Otherwise we
+            # compute it from the downloaded file.
+            # FIXME: https://github.com/pypa/pip/issues/11943
+            if (
+                isinstance(req.download_info.info, ArchiveInfo)
+                and not req.download_info.info.hashes
+                and req.local_file_path
+            ):
+                hash = hash_file(req.local_file_path)[0].hexdigest()
+                # We populate info.hash for backward compatibility.
+                # This will automatically populate info.hashes.
+                req.download_info.info.hash = f"sha256={hash}"
+
     def save_linked_requirement(self, req: InstallRequirement) -> None:
         assert self.download_dir is not None
         assert req.link is not None
+        assert req.is_concrete
         link = req.link
         if link.is_vcs or (link.is_existing_dir() and req.editable):
             # Make a .zip of the source_dir we already created.
@@ -703,6 +762,8 @@ class RequirementPreparer:
 
             req.check_if_exists(self.use_user_site)
 
+        # This should already have been populated by the preparation of the source dist.
+        assert req.is_concrete
         return dist
 
     def prepare_installed_requirement(
@@ -727,4 +788,13 @@ class RequirementPreparer:
                     "completely repeatable environment, install into an "
                     "empty virtualenv."
                 )
-            return InstalledDistribution(req).get_metadata_distribution()
+            dist = _get_prepared_distribution(
+                req,
+                self.build_tracker,
+                self.finder,
+                self.build_isolation,
+                self.check_build_deps,
+            )
+
+        assert req.is_concrete
+        return dist

src/pip/_internal/req/req_install.py

@@ -23,10 +23,7 @@ from pip._internal.locations import get_scheme
 from pip._internal.metadata import (
     BaseDistribution,
     get_default_environment,
-    get_directory_distribution,
-    get_wheel_distribution,
 )
-from pip._internal.metadata.base import FilesystemWheel
 from pip._internal.models.direct_url import DirectUrl
 from pip._internal.models.link import Link
 from pip._internal.operations.build.metadata import generate_metadata
@@ -150,6 +147,7 @@ class InstallRequirement:
         self.hash_options = hash_options if hash_options else {}
         self.config_settings = config_settings
         # Set to True after successful preparation of this requirement
+        # TODO: this is only used in the legacy resolver: remove this!
         self.prepared = False
         # User supplied requirement are explicitly requested for installation
         # by the user via CLI arguments or requirements files, as opposed to,
@@ -181,8 +179,13 @@ class InstallRequirement:
         # but after loading this flag should be treated as read only.
         self.use_pep517 = use_pep517
 
-        # This requirement needs more preparation before it can be built
-        self.needs_more_preparation = False
+        # When a dist is computed for this requirement, cache it here so it's visible
+        # everywhere within pip and isn't computed more than once. This may be
+        # a "virtual" dist without a physical location on the filesystem, or
+        # a "concrete" dist which has been fully downloaded.
+        self._cached_dist: Optional[BaseDistribution] = None
+        # Strictly used in testing: allow calling .cache_concrete_dist() twice.
+        self.allow_concrete_dist_overwrite = False
 
         # This requirement needs to be unpacked before it can be installed.
         self._archive_source: Optional[Path] = None
@@ -234,7 +237,7 @@ class InstallRequirement:
             return None
         return self.req.name
 
-    @functools.lru_cache()  # use cached_property in python 3.8+
+    @functools.lru_cache(maxsize=None)  # TODO: use cached_property in python 3.8+
     def supports_pyproject_editable(self) -> bool:
         if not self.use_pep517:
             return False
@@ -552,11 +555,11 @@ class InstallRequirement:
                 f"Consider using a build backend that supports PEP 660."
             )
 
-    def prepare_metadata(self) -> None:
+    def prepare_metadata_directory(self) -> None:
         """Ensure that project metadata is available.
 
-        Under PEP 517 and PEP 660, call the backend hook to prepare the metadata.
-        Under legacy processing, call setup.py egg-info.
+        Under PEP 517 and PEP 660, call the backend hook to prepare the metadata
+        directory.  Under legacy processing, call setup.py egg-info.
         """
         assert self.source_dir, f"No source dir for {self}"
         details = self.name or f"from {self.link}"
@@ -588,6 +591,8 @@ class InstallRequirement:
                 details=details,
             )
 
+    def validate_sdist_metadata(self) -> None:
+        """Ensure that we have a dist, and ensure it corresponds to expectations."""
         # Act on the newly generated metadata, based on the name and version.
         if not self.name:
             self._set_requirement()
@@ -598,24 +603,59 @@ class InstallRequirement:
 
     @property
     def metadata(self) -> Any:
+        # TODO: use cached_property in python 3.8+
         if not hasattr(self, "_metadata"):
-            self._metadata = self.get_dist().metadata
+            self._metadata = self.cached_dist.metadata
 
         return self._metadata
 
-    def get_dist(self) -> BaseDistribution:
-        if self.metadata_directory:
-            return get_directory_distribution(self.metadata_directory)
-        elif self.local_file_path and self.is_wheel:
-            assert self.req is not None
-            return get_wheel_distribution(
-                FilesystemWheel(self.local_file_path),
-                canonicalize_name(self.req.name),
+    @property
+    def cached_dist(self) -> BaseDistribution:
+        """Retrieve the dist resolved from this requirement.
+
+        :raises AssertionError: if the resolver has not yet been executed.
+        """
+        if self._cached_dist is None:
+            raise AssertionError(
+                f"InstallRequirement {self} has no dist; "
+                "ensure the resolver has been executed"
             )
-        raise AssertionError(
-            f"InstallRequirement {self} has no metadata directory and no wheel: "
-            f"can't make a distribution."
-        )
+        return self._cached_dist
+
+    def cache_virtual_metadata_only_dist(self, dist: BaseDistribution) -> None:
+        """Associate a "virtual" metadata-only dist to this requirement.
+
+        This dist cannot be installed, but it can be used to complete the resolve
+        process.
+
+        :raises AssertionError: if a dist has already been associated.
+        :raises AssertionError: if the provided dist is "concrete", i.e. exists
+                                somewhere on the filesystem.
+        """
+        assert self._cached_dist is None, self
+        assert not dist.is_concrete, dist
+        self._cached_dist = dist
+
+    def cache_concrete_dist(self, dist: BaseDistribution) -> None:
+        """Associate a "concrete" dist to this requirement.
+
+        A concrete dist exists somewhere on the filesystem and can be installed.
+
+        :raises AssertionError: if a concrete dist has already been associated.
+        :raises AssertionError: if the provided dist is not concrete.
+        """
+        if self._cached_dist is not None:
+            # If we set a dist twice for the same requirement, we must be hydrating
+            # a concrete dist for what was previously virtual. This will occur in the
+            # case of `install --dry-run` when PEP 658 metadata is available.
+            if not self.allow_concrete_dist_overwrite:
+                assert not self._cached_dist.is_concrete
+        assert dist.is_concrete
+        self._cached_dist = dist
+
+    @property
+    def is_concrete(self) -> bool:
+        return self._cached_dist is not None and self._cached_dist.is_concrete
 
     def assert_source_matches_version(self) -> None:
         assert self.source_dir, f"No source dir for {self}"

src/pip/_internal/req/req_set.py

@@ -86,7 +86,7 @@ class RequirementSet:
 
     def warn_legacy_versions_and_specifiers(self) -> None:
         for req in self.requirements_to_install:
-            version = req.get_dist().version
+            version = req.cached_dist.version
             if isinstance(version, LegacyVersion):
                 deprecated(
                     reason=(
@@ -101,7 +101,7 @@ class RequirementSet:
                     issue=12063,
                     gone_in="24.0",
                 )
-            for dep in req.get_dist().iter_dependencies():
+            for dep in req.cached_dist.iter_dependencies():
                 if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier):
                     deprecated(
                         reason=(

src/pip/_internal/resolution/resolvelib/resolver.py

@@ -175,11 +175,6 @@ class Resolver(BaseResolver):
 
             req_set.add_named_requirement(ireq)
 
-        reqs = req_set.all_requirements
-        self.factory.preparer.prepare_linked_requirements_more(reqs)
-        for req in reqs:
-            req.prepared = True
-            req.needs_more_preparation = False
         return req_set
 
     def get_installation_order(

tests/conftest.py

@@ -709,6 +709,9 @@ class FakePackage:
     requires_dist: Tuple[str, ...] = ()
     # This will override the Name specified in the actual dist's METADATA.
     metadata_name: Optional[str] = None
+    # Whether to delete the file this points to, which causes any attempt to fetch this
+    # package to fail unless it is processed as a metadata-only dist.
+    delete_linked_file: bool = False
 
     def metadata_filename(self) -> str:
         """This is specified by PEP 658."""
@@ -798,6 +801,27 @@ def fake_packages() -> Dict[str, List[FakePackage]]:
                 ("simple==1.0",),
             ),
         ],
+        "complex-dist": [
+            FakePackage(
+                "complex-dist",
+                "0.1",
+                "complex_dist-0.1-py2.py3-none-any.whl",
+                MetadataKind.Unhashed,
+                # Validate that the wheel isn't fetched if metadata is available and
+                # --dry-run is on, when the metadata presents no hash itself.
+                delete_linked_file=True,
+            ),
+        ],
+        "corruptwheel": [
+            FakePackage(
+                "corruptwheel",
+                "1.0",
+                "corruptwheel-1.0-py2.py3-none-any.whl",
+                # Validate that the wheel isn't fetched if metadata is available and
+                # --dry-run is on, when the metadata *does* present a hash.
+                MetadataKind.Sha256,
+            ),
+        ],
         "has-script": [
             # Ensure we check PEP 658 metadata hashing errors for wheel files.
             FakePackage(
@@ -883,10 +907,10 @@ def html_index_for_packages(
                 f'    <a href="{package_link.filename}" {package_link.generate_additional_tag()}>{package_link.filename}</a><br/>'  # noqa: E501
             )
             # (3.2) Copy over the corresponding file in `shared_data.packages`.
-            shutil.copy(
-                shared_data.packages / package_link.filename,
-                pkg_subdir / package_link.filename,
-            )
+            cached_file = shared_data.packages / package_link.filename
+            new_file = pkg_subdir / package_link.filename
+            if not package_link.delete_linked_file:
+                shutil.copy(cached_file, new_file)
             # (3.3) Write a metadata file, if applicable.
             if package_link.metadata != MetadataKind.NoFile:
                 with open(pkg_subdir / package_link.metadata_filename(), "wb") as f:

tests/functional/test_install_metadata.py

@@ -0,0 +1,236 @@
+import json
+import re
+from pathlib import Path
+from typing import Any, Callable, Dict, Iterator, List, Tuple
+
+import pytest
+from pip._vendor.packaging.requirements import Requirement
+
+from pip._internal.models.direct_url import DirectUrl
+from pip._internal.utils.urls import path_to_url
+from tests.lib import (
+    PipTestEnvironment,
+    TestPipResult,
+)
+
+
+@pytest.fixture(scope="function")
+def install_with_generated_html_index(
+    script: PipTestEnvironment,
+    html_index_for_packages: Path,
+    tmpdir: Path,
+) -> Callable[..., Tuple[TestPipResult, Dict[str, Any]]]:
+    """Execute `pip download` against a generated PyPI index."""
+    output_file = tmpdir / "output_file.json"
+
+    def run_for_generated_index(
+        args: List[str],
+        *,
+        dry_run: bool = True,
+        allow_error: bool = False,
+    ) -> Tuple[TestPipResult, Dict[str, Any]]:
+        """
+        Produce a PyPI directory structure pointing to the specified packages, then
+        execute `pip install --report ... -i ...` pointing to our generated index.
+        """
+        pip_args = [
+            "install",
+            *(("--dry-run",) if dry_run else ()),
+            "--ignore-installed",
+            "--report",
+            str(output_file),
+            "-i",
+            path_to_url(str(html_index_for_packages)),
+            *args,
+        ]
+        result = script.pip(*pip_args, allow_error=allow_error)
+        try:
+            with open(output_file, "rb") as f:
+                report = json.load(f)
+        except FileNotFoundError:
+            if allow_error:
+                report = {}
+            else:
+                raise
+        return (result, report)
+
+    return run_for_generated_index
+
+
+def iter_dists(report: Dict[str, Any]) -> Iterator[Tuple[Requirement, DirectUrl]]:
+    """Parse a (req,url) tuple from each installed dist in the --report json."""
+    for inst in report["install"]:
+        metadata = inst["metadata"]
+        name = metadata["name"]
+        version = metadata["version"]
+        req = Requirement(f"{name}=={version}")
+        direct_url = DirectUrl.from_dict(inst["download_info"])
+        yield (req, direct_url)
+
+
+@pytest.mark.parametrize(
+    "requirement_to_install, expected_outputs",
+    [
+        ("simple2==1.0", ["simple2==1.0", "simple==1.0"]),
+        ("simple==2.0", ["simple==2.0"]),
+        (
+            "colander",
+            ["colander==0.9.9", "translationstring==1.1"],
+        ),
+        (
+            "compilewheel",
+            ["compilewheel==1.0", "simple==1.0"],
+        ),
+    ],
+)
+def test_install_with_metadata(
+    install_with_generated_html_index: Callable[
+        ..., Tuple[TestPipResult, Dict[str, Any]]
+    ],
+    requirement_to_install: str,
+    expected_outputs: List[str],
+) -> None:
+    """Verify that if a data-dist-info-metadata attribute is present, then it is used
+    instead of the actual dist's METADATA."""
+    _, report = install_with_generated_html_index(
+        [requirement_to_install],
+    )
+    installed = sorted(str(r) for r, _ in iter_dists(report))
+    assert installed == expected_outputs
+
+
+@pytest.mark.parametrize(
+    "requirement_to_install, real_hash",
+    [
+        (
+            "simple==3.0",
+            "95e0f200b6302989bcf2cead9465cf229168295ea330ca30d1ffeab5c0fed996",
+        ),
+        (
+            "has-script",
+            "16ba92d7f6f992f6de5ecb7d58c914675cf21f57f8e674fb29dcb4f4c9507e5b",
+        ),
+    ],
+)
+def test_incorrect_metadata_hash(
+    install_with_generated_html_index: Callable[
+        ..., Tuple[TestPipResult, Dict[str, Any]]
+    ],
+    requirement_to_install: str,
+    real_hash: str,
+) -> None:
+    """Verify that if a hash for data-dist-info-metadata is provided, it must match the
+    actual hash of the metadata file."""
+    result, _ = install_with_generated_html_index(
+        [requirement_to_install],
+        allow_error=True,
+    )
+    assert result.returncode != 0
+    expected_msg = f"""\
+        Expected sha256 WRONG-HASH
+             Got        {real_hash}"""
+    assert expected_msg in result.stderr
+
+
+@pytest.mark.parametrize(
+    "requirement_to_install, expected_url",
+    [
+        ("simple2==2.0", "simple2-2.0.tar.gz.metadata"),
+        ("priority", "priority-1.0-py2.py3-none-any.whl.metadata"),
+    ],
+)
+def test_metadata_not_found(
+    install_with_generated_html_index: Callable[
+        ..., Tuple[TestPipResult, Dict[str, Any]]
+    ],
+    requirement_to_install: str,
+    expected_url: str,
+) -> None:
+    """Verify that if a data-dist-info-metadata attribute is provided, that pip will
+    fetch the .metadata file at the location specified by PEP 658, and error
+    if unavailable."""
+    result, _ = install_with_generated_html_index(
+        [requirement_to_install],
+        allow_error=True,
+    )
+    assert result.returncode != 0
+    expected_re = re.escape(expected_url)
+    pattern = re.compile(
+        f"ERROR: 404 Client Error: FileNotFoundError for url:.*{expected_re}"
+    )
+    assert pattern.search(result.stderr), (pattern, result.stderr)
+
+
+def test_produces_error_for_mismatched_package_name_in_metadata(
+    install_with_generated_html_index: Callable[
+        ..., Tuple[TestPipResult, Dict[str, Any]]
+    ],
+) -> None:
+    """Verify that the package name from the metadata matches the requested package."""
+    result, _ = install_with_generated_html_index(
+        ["simple2==3.0"],
+        allow_error=True,
+    )
+    assert result.returncode != 0
+    assert (
+        "simple2-3.0.tar.gz has inconsistent Name: expected 'simple2', but metadata "
+        "has 'not-simple2'"
+    ) in result.stdout
+
+
+@pytest.mark.parametrize(
+    "requirement",
+    (
+        "requires-simple-extra==0.1",
+        "REQUIRES_SIMPLE-EXTRA==0.1",
+        "REQUIRES....simple-_-EXTRA==0.1",
+    ),
+)
+def test_canonicalizes_package_name_before_verifying_metadata(
+    install_with_generated_html_index: Callable[
+        ..., Tuple[TestPipResult, Dict[str, Any]]
+    ],
+    requirement: str,
+) -> None:
+    """Verify that the package name from the command line and the package's
+    METADATA are both canonicalized before comparison, while the name from the METADATA
+    is always used verbatim to represent the installed candidate in --report.
+
+    Regression test for https://github.com/pypa/pip/issues/12038
+    """
+    _, report = install_with_generated_html_index(
+        [requirement],
+    )
+    reqs = [str(r) for r, _ in iter_dists(report)]
+    assert reqs == ["Requires_Simple.Extra==0.1"]
+
+
+@pytest.mark.parametrize(
+    "requirement,err_string",
+    (
+        # It's important that we verify pip won't even attempt to fetch the file, so we
+        # construct an input that will cause it to error if it tries at all.
+        ("complex-dist==0.1", "404 Client Error: FileNotFoundError"),
+        ("corruptwheel==1.0", ".whl is invalid."),
+    ),
+)
+def test_dry_run_avoids_downloading_metadata_only_dists(
+    install_with_generated_html_index: Callable[
+        ..., Tuple[TestPipResult, Dict[str, Any]]
+    ],
+    requirement: str,
+    err_string: str,
+) -> None:
+    """Verify that the underlying dist files are not downloaded at all when
+    `install --dry-run` is used to resolve dists with PEP 658 metadata."""
+    _, report = install_with_generated_html_index(
+        [requirement],
+    )
+    assert [requirement] == [str(r) for r, _ in iter_dists(report)]
+    result, _ = install_with_generated_html_index(
+        [requirement],
+        dry_run=False,
+        allow_error=True,
+    )
+    assert result.returncode != 0
+    assert err_string in result.stderr

tests/unit/metadata/test_metadata_pkg_resources.py

@@ -102,6 +102,7 @@ def test_wheel_metadata_works() -> None:
             metadata=InMemoryMetadata({"METADATA": metadata.as_bytes()}, "<in-memory>"),
             project_name=name,
         ),
+        concrete=False,
     )
 
     assert name == dist.canonical_name == dist.raw_name

tests/unit/test_req.py

@@ -154,6 +154,7 @@ class TestRequirementSet:
             os.fspath(data.packages.joinpath("LocalEnvironMarker")),
         )
         req.user_supplied = True
+        req.allow_concrete_dist_overwrite = True
         reqset.add_unnamed_requirement(req)
         finder = make_test_finder(find_links=[data.find_links])
         with self._basic_resolver(finder) as resolver:
@@ -503,6 +504,7 @@ class TestRequirementSet:
         with self._basic_resolver(finder) as resolver:
             ireq_url = data.packages.joinpath("FSPkg").as_uri()
             ireq = get_processed_req_from_line(f"-e {ireq_url}#egg=FSPkg")
+            ireq.allow_concrete_dist_overwrite = True
             reqset = resolver.resolve([ireq], True)
             assert len(reqset.all_requirements) == 1
             req = reqset.all_requirements[0]