Build local directories in-place with feature flag

docs/html/reference/pip_install.rst

@@ -808,7 +808,15 @@ You can install local projects by specifying the project path to pip:
 During regular installation, pip will copy the entire project directory to a
 temporary location and install from there. The exception is that pip will
 exclude .tox and .nox directories present in the top level of the project from
-being copied.
+being copied. This approach is the cause of several performance and correctness
+issues, so it is planned that pip 21.3 will change to install directly from the
+local project directory. Depending on the build backend used by the project,
+this may generate secondary build artifacts in the project directory, such as
+the ``.egg-info`` and ``build`` directories in the case of the setuptools
+backend.
+
+To opt in to the future behavior, specify the ``--use-feature=in-tree-build``
+option in pip's command line.
 
 
 .. _`editable-installs`:

news/9091.feature.rst

@@ -0,0 +1,4 @@
+Add a feature ``--use-feature=in-tree-build`` to build local projects in-place
+when installing. This is expected to become the default behavior in pip 21.3;
+see `Installing from local packages <https://pip.pypa.io/en/stable/user_guide/#installing-from-local-packages>`_
+for more information.

src/pip/_internal/cli/cmdoptions.py

@@ -951,7 +951,7 @@ use_new_feature = partial(
     metavar="feature",
     action="append",
     default=[],
-    choices=["2020-resolver", "fast-deps"],
+    choices=["2020-resolver", "fast-deps", "in-tree-build"],
     help="Enable new functionality, that may be backward incompatible.",
 )  # type: Callable[..., Option]
 

src/pip/_internal/cli/req_command.py

@@ -245,6 +245,7 @@ class RequirementCommand(IndexGroupCommand):
             require_hashes=options.require_hashes,
             use_user_site=use_user_site,
             lazy_wheel=lazy_wheel,
+            in_tree_build="in-tree-build" in options.features_enabled,
         )
 
     @classmethod

src/pip/_internal/operations/prepare.py

@@ -35,6 +35,7 @@ from pip._internal.network.lazy_wheel import (
 from pip._internal.network.session import PipSession
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.req.req_tracker import RequirementTracker
+from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.filesystem import copy2_fixed
 from pip._internal.utils.hashes import Hashes, MissingHashes
 from pip._internal.utils.logging import indent_log
@@ -207,8 +208,23 @@ def unpack_url(
         unpack_vcs_link(link, location)
         return None
 
-    # If it's a url to a local directory
+    # Once out-of-tree-builds are no longer supported, could potentially
+    # replace the below condition with `assert not link.is_existing_dir`
+    # - unpack_url does not need to be called for in-tree-builds.
+    #
+    # As further cleanup, _copy_source_tree and accompanying tests can
+    # be removed.
     if link.is_existing_dir():
+        deprecated(
+            "A future pip version will change local packages to be built "
+            "in-place without first copying to a temporary directory. "
+            "We recommend you use --use-feature=in-tree-build to test "
+            "your packages with this new behavior before it becomes the "
+            "default.\n",
+            replacement=None,
+            gone_in="21.3",
+            issue=7555
+        )
         if os.path.isdir(location):
             rmtree(location)
         _copy_source_tree(link.file_path, location)
@@ -278,6 +294,7 @@ class RequirementPreparer:
         require_hashes,  # type: bool
         use_user_site,  # type: bool
         lazy_wheel,  # type: bool
+        in_tree_build,  # type: bool
     ):
         # type: (...) -> None
         super().__init__()
@@ -306,6 +323,9 @@ class RequirementPreparer:
         # Should wheels be downloaded lazily?
         self.use_lazy_wheel = lazy_wheel
 
+        # Should in-tree builds be used for local paths?
+        self.in_tree_build = in_tree_build
+
         # Memoized downloaded files, as mapping of url: (path, mime type)
         self._downloaded = {}  # type: Dict[str, Tuple[str, str]]
 
@@ -339,6 +359,11 @@ class RequirementPreparer:
             # directory.
             return
         assert req.source_dir is None
+        if req.link.is_existing_dir() and self.in_tree_build:
+            # build local directories in-tree
+            req.source_dir = req.link.file_path
+            return
+
         # We always delete unpacked sdists after pip runs.
         req.ensure_has_source_dir(
             self.build_dir,
@@ -517,11 +542,14 @@ class RequirementPreparer:
 
         self._ensure_link_req_src_dir(req, parallel_builds)
         hashes = self._get_linked_req_hashes(req)
-        if link.url not in self._downloaded:
+
+        if link.is_existing_dir() and self.in_tree_build:
+            local_file = None
+        elif link.url not in self._downloaded:
             try:
                 local_file = unpack_url(
                     link, req.source_dir, self._download,
-                    self.download_dir, hashes,
+                    self.download_dir, hashes
                 )
             except NetworkConnectionError as exc:
                 raise InstallationError(

tests/functional/test_install.py

@@ -581,6 +581,28 @@ def test_install_from_local_directory_with_symlinks_to_directories(
     result.did_create(dist_info_folder)
 
 
+def test_install_from_local_directory_with_in_tree_build(
+    script, data, with_wheel
+):
+    """
+    Test installing from a local directory with --use-feature=in-tree-build.
+    """
+    to_install = data.packages.joinpath("FSPkg")
+    args = ["install", "--use-feature=in-tree-build", to_install]
+
+    in_tree_build_dir = to_install / "build"
+    assert not in_tree_build_dir.exists()
+    result = script.pip(*args)
+    fspkg_folder = script.site_packages / 'fspkg'
+    dist_info_folder = (
+        script.site_packages /
+        'FSPkg-0.1.dev0.dist-info'
+    )
+    result.did_create(fspkg_folder)
+    result.did_create(dist_info_folder)
+    assert in_tree_build_dir.exists()
+
+
 @pytest.mark.skipif("sys.platform == 'win32' or sys.version_info < (3,)")
 def test_install_from_local_directory_with_socket_file(
     script, data, tmpdir, with_wheel

tests/unit/test_req.py

@@ -89,6 +89,7 @@ class TestRequirementSet:
                 require_hashes=require_hashes,
                 use_user_site=False,
                 lazy_wheel=False,
+                in_tree_build=False,
             )
             yield Resolver(
                 preparer=preparer,