diff --git a/docs/html/topics/local-project-installs.md b/docs/html/topics/local-project-installs.md index 331afadb8..151035b00 100644 --- a/docs/html/topics/local-project-installs.md +++ b/docs/html/topics/local-project-installs.md @@ -51,15 +51,17 @@ There are two advantages over using `setup.py develop` directly: ## Build artifacts ```{versionchanged} 21.3 -The project being installed is no longer copied to a temporary directory before invoking the build system. +The project being installed is no longer copied to a temporary directory before invoking the build system, by default. A `--use-deprecated=out-of-tree-build` option is provided as a temporary fallback to aid user migrations. ``` -This behaviour change has several consequences: +```{versionchanged} 22.1 +The `--use-deprecated=out-of-tree-build` option has been removed. +``` + +When provided with a project that's in a local directory, pip will invoke the build system "in place". This behaviour has several consequences: - Local project builds will now be significantly faster, for certain kinds of projects and on systems with slow I/O (eg: via network attached storage or overly aggressive antivirus software). - Certain build backends (eg: `setuptools`) will litter the project directory with secondary build artifacts (eg: `.egg-info` directories). - Certain build backends (eg: `setuptools`) may not be able to perform with parallel builds anymore, since they previously relied on the fact that pip invoked them in a separate directory for each build. -A `--use-deprecated=out-of-tree-build` option is available, until pip 22.1, as a mechanism to aid users with transitioning to the newer model of in-tree-builds. - [^1]: Specifically, the current machine's filesystem. diff --git a/news/11001.removal.rst b/news/11001.removal.rst new file mode 100644 index 000000000..a0dac9018 --- /dev/null +++ b/news/11001.removal.rst @@ -0,0 +1 @@ +Drop ``--use-deprecated=out-of-tree-build``, according to deprecation message. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index dcfbd9bb2..3612dc456 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -957,7 +957,7 @@ use_new_feature: Callable[..., Option] = partial( metavar="feature", action="append", default=[], - choices=["2020-resolver", "fast-deps", "in-tree-build"], + choices=["2020-resolver", "fast-deps"], help="Enable new functionality, that may be backward incompatible.", ) @@ -970,7 +970,6 @@ use_deprecated_feature: Callable[..., Option] = partial( default=[], choices=[ "legacy-resolver", - "out-of-tree-build", "backtrack-on-build-failures", "html5lib", ], diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index cbb17c5f0..2c2db00de 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -288,20 +288,6 @@ class RequirementCommand(IndexGroupCommand): "fast-deps has no effect when used with the legacy resolver." ) - in_tree_build = "out-of-tree-build" not in options.deprecated_features_enabled - if "in-tree-build" in options.features_enabled: - deprecated( - reason="In-tree builds are now the default.", - replacement="to remove the --use-feature=in-tree-build flag", - gone_in="22.1", - ) - if "out-of-tree-build" in options.deprecated_features_enabled: - deprecated( - reason="Out-of-tree builds are deprecated.", - replacement=None, - gone_in="22.1", - ) - return RequirementPreparer( build_dir=temp_build_dir_path, src_dir=options.src_dir, @@ -315,7 +301,6 @@ class RequirementCommand(IndexGroupCommand): use_user_site=use_user_site, lazy_wheel=lazy_wheel, verbosity=verbosity, - in_tree_build=in_tree_build, ) @classmethod diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 46252816d..7550d3a90 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -35,10 +35,9 @@ from pip._internal.network.lazy_wheel import ( from pip._internal.network.session import PipSession from pip._internal.operations.build.build_tracker import BuildTracker from pip._internal.req.req_install import InstallRequirement -from pip._internal.utils.filesystem import copy2_fixed from pip._internal.utils.hashes import Hashes, MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, hide_url, is_installable_dir, rmtree +from pip._internal.utils.misc import display_path, hide_url, is_installable_dir from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs @@ -98,55 +97,6 @@ def get_http_url( return File(from_path, content_type) -def _copy2_ignoring_special_files(src: str, dest: str) -> None: - """Copying special files is not supported, but as a convenience to users - we skip errors copying them. This supports tools that may create e.g. - socket files in the project source directory. - """ - try: - copy2_fixed(src, dest) - except shutil.SpecialFileError as e: - # SpecialFileError may be raised due to either the source or - # destination. If the destination was the cause then we would actually - # care, but since the destination directory is deleted prior to - # copy we ignore all of them assuming it is caused by the source. - logger.warning( - "Ignoring special file error '%s' encountered copying %s to %s.", - str(e), - src, - dest, - ) - - -def _copy_source_tree(source: str, target: str) -> None: - target_abspath = os.path.abspath(target) - target_basename = os.path.basename(target_abspath) - target_dirname = os.path.dirname(target_abspath) - - def ignore(d: str, names: List[str]) -> List[str]: - skipped: List[str] = [] - if d == source: - # Pulling in those directories can potentially be very slow, - # exclude the following directories if they appear in the top - # level dir (and only it). - # See discussion at https://github.com/pypa/pip/pull/6770 - skipped += [".tox", ".nox"] - if os.path.abspath(d) == target_dirname: - # Prevent an infinite recursion if the target is in source. - # This can happen when TMPDIR is set to ${PWD}/... - # and we copy PWD to TMPDIR. - skipped += [target_basename] - return skipped - - shutil.copytree( - source, - target, - ignore=ignore, - symlinks=True, - copy_function=_copy2_ignoring_special_files, - ) - - def get_file_url( link: Link, download_dir: Optional[str] = None, hashes: Optional[Hashes] = None ) -> File: @@ -191,19 +141,7 @@ def unpack_url( unpack_vcs_link(link, location, verbosity=verbosity) return None - # Once out-of-tree-builds are no longer supported, could potentially - # replace the below condition with `assert not link.is_existing_dir` - # - unpack_url does not need to be called for in-tree-builds. - # - # As further cleanup, _copy_source_tree and accompanying tests can - # be removed. - # - # TODO when use-deprecated=out-of-tree-build is removed - if link.is_existing_dir(): - if os.path.isdir(location): - rmtree(location) - _copy_source_tree(link.file_path, location) - return None + assert not link.is_existing_dir() # file urls if link.is_file: @@ -269,7 +207,6 @@ class RequirementPreparer: use_user_site: bool, lazy_wheel: bool, verbosity: int, - in_tree_build: bool, ) -> None: super().__init__() @@ -300,9 +237,6 @@ class RequirementPreparer: # How verbose should underlying tooling be? self.verbosity = verbosity - # Should in-tree builds be used for local paths? - self.in_tree_build = in_tree_build - # Memoized downloaded files, as mapping of url: path. self._downloaded: Dict[str, str] = {} @@ -336,7 +270,7 @@ class RequirementPreparer: # directory. return assert req.source_dir is None - if req.link.is_existing_dir() and self.in_tree_build: + if req.link.is_existing_dir(): # build local directories in-tree req.source_dir = req.link.file_path return @@ -525,7 +459,7 @@ class RequirementPreparer: self._ensure_link_req_src_dir(req, parallel_builds) hashes = self._get_linked_req_hashes(req) - if link.is_existing_dir() and self.in_tree_build: + if link.is_existing_dir(): local_file = None elif link.url not in self._downloaded: try: diff --git a/src/pip/_internal/utils/filesystem.py b/src/pip/_internal/utils/filesystem.py index b7e6191ab..ccf298d0e 100644 --- a/src/pip/_internal/utils/filesystem.py +++ b/src/pip/_internal/utils/filesystem.py @@ -2,8 +2,6 @@ import fnmatch import os import os.path import random -import shutil -import stat import sys from contextlib import contextmanager from tempfile import NamedTemporaryFile @@ -42,33 +40,6 @@ def check_path_owner(path: str) -> bool: return False # assume we don't own the path -def copy2_fixed(src: str, dest: str) -> None: - """Wrap shutil.copy2() but map errors copying socket files to - SpecialFileError as expected. - - See also https://bugs.python.org/issue37700. - """ - try: - shutil.copy2(src, dest) - except OSError: - for f in [src, dest]: - try: - is_socket_file = is_socket(f) - except OSError: - # An error has already occurred. Another error here is not - # a problem and we can ignore it. - pass - else: - if is_socket_file: - raise shutil.SpecialFileError(f"`{f}` is a socket") - - raise - - -def is_socket(path: str) -> bool: - return stat.S_ISSOCK(os.lstat(path).st_mode) - - @contextmanager def adjacent_tmp_file(path: str, **kwargs: Any) -> Iterator[BinaryIO]: """Return a file-like object pointing to a tmp file next to path. diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index e089a8f69..bec8b72fc 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -2,7 +2,6 @@ import distutils import glob import os import re -import shutil import ssl import sys import textwrap @@ -30,7 +29,6 @@ from tests.lib import ( pyversion, requirements_file, ) -from tests.lib.filesystem import make_socket_file from tests.lib.local_repos import local_checkout from tests.lib.path import Path from tests.lib.server import ( @@ -648,26 +646,6 @@ def test_hashed_install_failure_later_flag( ) -@pytest.mark.usefixtures("with_wheel") -def test_install_from_local_directory_with_symlinks_to_directories( - script: PipTestEnvironment, data: TestData -) -> None: - """ - Test installing from a local directory containing symlinks to directories. - """ - to_install = data.packages.joinpath("symlinks") - result = script.pip( - "install", - "--use-deprecated=out-of-tree-build", - to_install, - allow_stderr_warning=True, # TODO: set to False when removing out-of-tree-build - ) - pkg_folder = script.site_packages / "symlinks" - dist_info_folder = script.site_packages / "symlinks-0.1.dev0.dist-info" - result.did_create(pkg_folder) - result.did_create(dist_info_folder) - - @pytest.mark.usefixtures("with_wheel") def test_install_from_local_directory_with_in_tree_build( script: PipTestEnvironment, data: TestData @@ -688,38 +666,6 @@ def test_install_from_local_directory_with_in_tree_build( assert in_tree_build_dir.exists() -@pytest.mark.skipif("sys.platform == 'win32'") -@pytest.mark.usefixtures("with_wheel") -def test_install_from_local_directory_with_socket_file( - script: PipTestEnvironment, data: TestData, tmpdir: Path -) -> None: - """ - Test installing from a local directory containing a socket file. - """ - # TODO: remove this test when removing out-of-tree-build support, - # it is only meant to test the copy of socket files - dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info" - package_folder = script.site_packages / "fspkg" - to_copy = data.packages.joinpath("FSPkg") - to_install = tmpdir.joinpath("src") - - shutil.copytree(to_copy, to_install) - # Socket file, should be ignored. - socket_file_path = os.path.join(to_install, "example") - make_socket_file(socket_file_path) - - result = script.pip( - "install", - "--use-deprecated=out-of-tree-build", - "--verbose", - to_install, - allow_stderr_warning=True, # because of the out-of-tree deprecation warning - ) - result.did_create(package_folder) - result.did_create(dist_info_folder) - assert str(socket_file_path) in result.stderr - - def test_install_from_local_directory_with_no_setup_py( script: PipTestEnvironment, data: TestData ) -> None: diff --git a/tests/lib/filesystem.py b/tests/lib/filesystem.py index 8563783e7..5f8fe519d 100644 --- a/tests/lib/filesystem.py +++ b/tests/lib/filesystem.py @@ -1,38 +1,10 @@ """Helpers for filesystem-dependent tests. """ import os -import socket -import subprocess -import sys from functools import partial from itertools import chain from typing import Iterator, List, Set -from .path import Path - - -def make_socket_file(path: str) -> None: - # Socket paths are limited to 108 characters (sometimes less) so we - # chdir before creating it and use a relative path name. - cwd = os.getcwd() - os.chdir(os.path.dirname(path)) - try: - sock = socket.socket(socket.AF_UNIX) - sock.bind(os.path.basename(path)) - finally: - os.chdir(cwd) - - -def make_unreadable_file(path: str) -> None: - Path(path).touch() - os.chmod(path, 0o000) - if sys.platform == "win32": - username = os.getlogin() - # Remove "Read Data/List Directory" permission for current user, but - # leave everything else. - args = ["icacls", path, "/deny", username + ":(RD)"] - subprocess.check_call(args) - def get_filelist(base: str) -> Set[str]: def join(dirpath: str, dirnames: List[str], filenames: List[str]) -> Iterator[str]: diff --git a/tests/unit/test_operations_prepare.py b/tests/unit/test_operations_prepare.py index a2ee87870..8838fa9ce 100644 --- a/tests/unit/test_operations_prepare.py +++ b/tests/unit/test_operations_prepare.py @@ -11,11 +11,10 @@ from pip._internal.exceptions import HashMismatch from pip._internal.models.link import Link from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession -from pip._internal.operations.prepare import _copy_source_tree, unpack_url +from pip._internal.operations.prepare import unpack_url from pip._internal.utils.hashes import Hashes from pip._internal.utils.urls import path_to_url from tests.lib import TestData -from tests.lib.filesystem import get_filelist, make_socket_file, make_unreadable_file from tests.lib.path import Path from tests.lib.requests_mocks import MockResponse @@ -99,80 +98,6 @@ def clean_project(tmpdir_factory: pytest.TempdirFactory, data: TestData) -> Path return new_project_dir -def test_copy_source_tree(clean_project: Path, tmpdir: Path) -> None: - target = tmpdir.joinpath("target") - expected_files = get_filelist(clean_project) - assert len(expected_files) == 3 - - _copy_source_tree(clean_project, target) - - copied_files = get_filelist(target) - assert expected_files == copied_files - - -@pytest.mark.skipif("sys.platform == 'win32'") -def test_copy_source_tree_with_socket( - clean_project: Path, tmpdir: Path, caplog: pytest.LogCaptureFixture -) -> None: - target = tmpdir.joinpath("target") - expected_files = get_filelist(clean_project) - socket_path = str(clean_project.joinpath("aaa")) - make_socket_file(socket_path) - - _copy_source_tree(clean_project, target) - - copied_files = get_filelist(target) - assert expected_files == copied_files - - # Warning should have been logged. - assert len(caplog.records) == 1 - record = caplog.records[0] - assert record.levelname == "WARNING" - assert socket_path in record.message - - -@pytest.mark.skipif("sys.platform == 'win32'") -def test_copy_source_tree_with_socket_fails_with_no_socket_error( - clean_project: Path, tmpdir: Path -) -> None: - target = tmpdir.joinpath("target") - expected_files = get_filelist(clean_project) - make_socket_file(clean_project.joinpath("aaa")) - unreadable_file = clean_project.joinpath("bbb") - make_unreadable_file(unreadable_file) - - with pytest.raises(shutil.Error) as e: - _copy_source_tree(clean_project, target) - - errored_files = [err[0] for err in e.value.args[0]] - assert len(errored_files) == 1 - assert unreadable_file in errored_files - - copied_files = get_filelist(target) - # All files without errors should have been copied. - assert expected_files == copied_files - - -def test_copy_source_tree_with_unreadable_dir_fails( - clean_project: Path, tmpdir: Path -) -> None: - target = tmpdir.joinpath("target") - expected_files = get_filelist(clean_project) - unreadable_file = clean_project.joinpath("bbb") - make_unreadable_file(unreadable_file) - - with pytest.raises(shutil.Error) as e: - _copy_source_tree(clean_project, target) - - errored_files = [err[0] for err in e.value.args[0]] - assert len(errored_files) == 1 - assert unreadable_file in errored_files - - copied_files = get_filelist(target) - # All files without errors should have been copied. - assert expected_files == copied_files - - class Test_unpack_url: def prep(self, tmpdir: Path, data: TestData) -> None: self.build_dir = tmpdir.joinpath("build") @@ -208,50 +133,3 @@ class Test_unpack_url: hashes=Hashes({"md5": ["bogus"]}), verbosity=0, ) - - def test_unpack_url_thats_a_dir(self, tmpdir: Path, data: TestData) -> None: - self.prep(tmpdir, data) - dist_path = data.packages.joinpath("FSPkg") - dist_url = Link(path_to_url(dist_path)) - unpack_url( - dist_url, - self.build_dir, - download=self.no_download, - download_dir=self.download_dir, - verbosity=0, - ) - assert os.path.isdir(os.path.join(self.build_dir, "fspkg")) - - -@pytest.mark.parametrize("exclude_dir", [".nox", ".tox"]) -def test_unpack_url_excludes_expected_dirs(tmpdir: Path, exclude_dir: str) -> None: - src_dir = tmpdir / "src" - dst_dir = tmpdir / "dst" - src_included_file = src_dir.joinpath("file.txt") - src_excluded_dir = src_dir.joinpath(exclude_dir) - src_excluded_file = src_dir.joinpath(exclude_dir, "file.txt") - src_included_dir = src_dir.joinpath("subdir", exclude_dir) - - # set up source directory - src_excluded_dir.mkdir(parents=True) - src_included_dir.mkdir(parents=True) - src_included_file.touch() - src_excluded_file.touch() - - dst_included_file = dst_dir.joinpath("file.txt") - dst_excluded_dir = dst_dir.joinpath(exclude_dir) - dst_excluded_file = dst_dir.joinpath(exclude_dir, "file.txt") - dst_included_dir = dst_dir.joinpath("subdir", exclude_dir) - - src_link = Link(path_to_url(src_dir)) - unpack_url( - src_link, - dst_dir, - Mock(side_effect=AssertionError), - download_dir=None, - verbosity=0, - ) - assert not os.path.isdir(dst_excluded_dir) - assert not os.path.isfile(dst_excluded_file) - assert os.path.isfile(dst_included_file) - assert os.path.isdir(dst_included_dir) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 4a339e4e2..075d12688 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -99,7 +99,6 @@ class TestRequirementSet: use_user_site=False, lazy_wheel=False, verbosity=0, - in_tree_build=False, ) yield Resolver( preparer=preparer, diff --git a/tests/unit/test_utils_filesystem.py b/tests/unit/test_utils_filesystem.py index b15c3141a..1b730f8b8 100644 --- a/tests/unit/test_utils_filesystem.py +++ b/tests/unit/test_utils_filesystem.py @@ -1,11 +1,5 @@ import os -import shutil -from typing import Callable, Type -import pytest - -from pip._internal.utils.filesystem import copy2_fixed, is_socket -from tests.lib.filesystem import make_socket_file, make_unreadable_file from tests.lib.path import Path @@ -25,44 +19,3 @@ def make_broken_symlink(path: str) -> None: def make_dir(path: str) -> None: os.mkdir(path) - - -skip_on_windows = pytest.mark.skipif("sys.platform == 'win32'") - - -@skip_on_windows -@pytest.mark.parametrize( - "create,result", - [ - (make_socket_file, True), - (make_file, False), - (make_valid_symlink, False), - (make_broken_symlink, False), - (make_dir, False), - ], -) -def test_is_socket(create: Callable[[str], None], result: bool, tmpdir: Path) -> None: - target = tmpdir.joinpath("target") - create(target) - assert os.path.lexists(target) - assert is_socket(target) == result - - -@pytest.mark.parametrize( - "create,error_type", - [ - pytest.param(make_socket_file, shutil.SpecialFileError, marks=skip_on_windows), - (make_unreadable_file, OSError), - ], -) -def test_copy2_fixed_raises_appropriate_errors( - create: Callable[[str], None], error_type: Type[Exception], tmpdir: Path -) -> None: - src = tmpdir.joinpath("src") - create(src) - dest = tmpdir.joinpath("dest") - - with pytest.raises(error_type): - copy2_fixed(src, dest) - - assert not dest.exists()