Drop out-of-tree/in-tree build transition flags

These were intended to help users transition when the default behaviour
changed to no longer perform out-of-tree builds. The transition is now
considered complete.
This commit is contained in:
Pradyun Gedam 2022-04-01 12:37:51 +01:00 committed by Pradyun Gedam
parent 7a4c3ba03a
commit 428e886ad6
No known key found for this signature in database
GPG Key ID: FF99710C4332258E
11 changed files with 13 additions and 373 deletions

View File

@ -51,15 +51,17 @@ There are two advantages over using `setup.py develop` directly:
## Build artifacts
```{versionchanged} 21.3
The project being installed is no longer copied to a temporary directory before invoking the build system.
The project being installed is no longer copied to a temporary directory before invoking the build system, by default. A `--use-deprecated=out-of-tree-build` option is provided as a temporary fallback to aid user migrations.
```
This behaviour change has several consequences:
```{versionchanged} 22.1
The `--use-deprecated=out-of-tree-build` option has been removed.
```
When provided with a project that's in a local directory, pip will invoke the build system "in place". This behaviour has several consequences:
- Local project builds will now be significantly faster, for certain kinds of projects and on systems with slow I/O (eg: via network attached storage or overly aggressive antivirus software).
- Certain build backends (eg: `setuptools`) will litter the project directory with secondary build artifacts (eg: `.egg-info` directories).
- Certain build backends (eg: `setuptools`) may not be able to perform with parallel builds anymore, since they previously relied on the fact that pip invoked them in a separate directory for each build.
A `--use-deprecated=out-of-tree-build` option is available, until pip 22.1, as a mechanism to aid users with transitioning to the newer model of in-tree-builds.
[^1]: Specifically, the current machine's filesystem.

1
news/11001.removal.rst Normal file
View File

@ -0,0 +1 @@
Drop ``--use-deprecated=out-of-tree-build``, according to deprecation message.

View File

@ -957,7 +957,7 @@ use_new_feature: Callable[..., Option] = partial(
metavar="feature",
action="append",
default=[],
choices=["2020-resolver", "fast-deps", "in-tree-build"],
choices=["2020-resolver", "fast-deps"],
help="Enable new functionality, that may be backward incompatible.",
)
@ -970,7 +970,6 @@ use_deprecated_feature: Callable[..., Option] = partial(
default=[],
choices=[
"legacy-resolver",
"out-of-tree-build",
"backtrack-on-build-failures",
"html5lib",
],

View File

@ -288,20 +288,6 @@ class RequirementCommand(IndexGroupCommand):
"fast-deps has no effect when used with the legacy resolver."
)
in_tree_build = "out-of-tree-build" not in options.deprecated_features_enabled
if "in-tree-build" in options.features_enabled:
deprecated(
reason="In-tree builds are now the default.",
replacement="to remove the --use-feature=in-tree-build flag",
gone_in="22.1",
)
if "out-of-tree-build" in options.deprecated_features_enabled:
deprecated(
reason="Out-of-tree builds are deprecated.",
replacement=None,
gone_in="22.1",
)
return RequirementPreparer(
build_dir=temp_build_dir_path,
src_dir=options.src_dir,
@ -315,7 +301,6 @@ class RequirementCommand(IndexGroupCommand):
use_user_site=use_user_site,
lazy_wheel=lazy_wheel,
verbosity=verbosity,
in_tree_build=in_tree_build,
)
@classmethod

View File

@ -35,10 +35,9 @@ from pip._internal.network.lazy_wheel import (
from pip._internal.network.session import PipSession
from pip._internal.operations.build.build_tracker import BuildTracker
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.filesystem import copy2_fixed
from pip._internal.utils.hashes import Hashes, MissingHashes
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import display_path, hide_url, is_installable_dir, rmtree
from pip._internal.utils.misc import display_path, hide_url, is_installable_dir
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.unpacking import unpack_file
from pip._internal.vcs import vcs
@ -98,55 +97,6 @@ def get_http_url(
return File(from_path, content_type)
def _copy2_ignoring_special_files(src: str, dest: str) -> None:
"""Copying special files is not supported, but as a convenience to users
we skip errors copying them. This supports tools that may create e.g.
socket files in the project source directory.
"""
try:
copy2_fixed(src, dest)
except shutil.SpecialFileError as e:
# SpecialFileError may be raised due to either the source or
# destination. If the destination was the cause then we would actually
# care, but since the destination directory is deleted prior to
# copy we ignore all of them assuming it is caused by the source.
logger.warning(
"Ignoring special file error '%s' encountered copying %s to %s.",
str(e),
src,
dest,
)
def _copy_source_tree(source: str, target: str) -> None:
target_abspath = os.path.abspath(target)
target_basename = os.path.basename(target_abspath)
target_dirname = os.path.dirname(target_abspath)
def ignore(d: str, names: List[str]) -> List[str]:
skipped: List[str] = []
if d == source:
# Pulling in those directories can potentially be very slow,
# exclude the following directories if they appear in the top
# level dir (and only it).
# See discussion at https://github.com/pypa/pip/pull/6770
skipped += [".tox", ".nox"]
if os.path.abspath(d) == target_dirname:
# Prevent an infinite recursion if the target is in source.
# This can happen when TMPDIR is set to ${PWD}/...
# and we copy PWD to TMPDIR.
skipped += [target_basename]
return skipped
shutil.copytree(
source,
target,
ignore=ignore,
symlinks=True,
copy_function=_copy2_ignoring_special_files,
)
def get_file_url(
link: Link, download_dir: Optional[str] = None, hashes: Optional[Hashes] = None
) -> File:
@ -191,19 +141,7 @@ def unpack_url(
unpack_vcs_link(link, location, verbosity=verbosity)
return None
# Once out-of-tree-builds are no longer supported, could potentially
# replace the below condition with `assert not link.is_existing_dir`
# - unpack_url does not need to be called for in-tree-builds.
#
# As further cleanup, _copy_source_tree and accompanying tests can
# be removed.
#
# TODO when use-deprecated=out-of-tree-build is removed
if link.is_existing_dir():
if os.path.isdir(location):
rmtree(location)
_copy_source_tree(link.file_path, location)
return None
assert not link.is_existing_dir()
# file urls
if link.is_file:
@ -269,7 +207,6 @@ class RequirementPreparer:
use_user_site: bool,
lazy_wheel: bool,
verbosity: int,
in_tree_build: bool,
) -> None:
super().__init__()
@ -300,9 +237,6 @@ class RequirementPreparer:
# How verbose should underlying tooling be?
self.verbosity = verbosity
# Should in-tree builds be used for local paths?
self.in_tree_build = in_tree_build
# Memoized downloaded files, as mapping of url: path.
self._downloaded: Dict[str, str] = {}
@ -336,7 +270,7 @@ class RequirementPreparer:
# directory.
return
assert req.source_dir is None
if req.link.is_existing_dir() and self.in_tree_build:
if req.link.is_existing_dir():
# build local directories in-tree
req.source_dir = req.link.file_path
return
@ -525,7 +459,7 @@ class RequirementPreparer:
self._ensure_link_req_src_dir(req, parallel_builds)
hashes = self._get_linked_req_hashes(req)
if link.is_existing_dir() and self.in_tree_build:
if link.is_existing_dir():
local_file = None
elif link.url not in self._downloaded:
try:

View File

@ -2,8 +2,6 @@ import fnmatch
import os
import os.path
import random
import shutil
import stat
import sys
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
@ -42,33 +40,6 @@ def check_path_owner(path: str) -> bool:
return False # assume we don't own the path
def copy2_fixed(src: str, dest: str) -> None:
"""Wrap shutil.copy2() but map errors copying socket files to
SpecialFileError as expected.
See also https://bugs.python.org/issue37700.
"""
try:
shutil.copy2(src, dest)
except OSError:
for f in [src, dest]:
try:
is_socket_file = is_socket(f)
except OSError:
# An error has already occurred. Another error here is not
# a problem and we can ignore it.
pass
else:
if is_socket_file:
raise shutil.SpecialFileError(f"`{f}` is a socket")
raise
def is_socket(path: str) -> bool:
return stat.S_ISSOCK(os.lstat(path).st_mode)
@contextmanager
def adjacent_tmp_file(path: str, **kwargs: Any) -> Iterator[BinaryIO]:
"""Return a file-like object pointing to a tmp file next to path.

View File

@ -2,7 +2,6 @@ import distutils
import glob
import os
import re
import shutil
import ssl
import sys
import textwrap
@ -30,7 +29,6 @@ from tests.lib import (
pyversion,
requirements_file,
)
from tests.lib.filesystem import make_socket_file
from tests.lib.local_repos import local_checkout
from tests.lib.path import Path
from tests.lib.server import (
@ -648,26 +646,6 @@ def test_hashed_install_failure_later_flag(
)
@pytest.mark.usefixtures("with_wheel")
def test_install_from_local_directory_with_symlinks_to_directories(
script: PipTestEnvironment, data: TestData
) -> None:
"""
Test installing from a local directory containing symlinks to directories.
"""
to_install = data.packages.joinpath("symlinks")
result = script.pip(
"install",
"--use-deprecated=out-of-tree-build",
to_install,
allow_stderr_warning=True, # TODO: set to False when removing out-of-tree-build
)
pkg_folder = script.site_packages / "symlinks"
dist_info_folder = script.site_packages / "symlinks-0.1.dev0.dist-info"
result.did_create(pkg_folder)
result.did_create(dist_info_folder)
@pytest.mark.usefixtures("with_wheel")
def test_install_from_local_directory_with_in_tree_build(
script: PipTestEnvironment, data: TestData
@ -688,38 +666,6 @@ def test_install_from_local_directory_with_in_tree_build(
assert in_tree_build_dir.exists()
@pytest.mark.skipif("sys.platform == 'win32'")
@pytest.mark.usefixtures("with_wheel")
def test_install_from_local_directory_with_socket_file(
script: PipTestEnvironment, data: TestData, tmpdir: Path
) -> None:
"""
Test installing from a local directory containing a socket file.
"""
# TODO: remove this test when removing out-of-tree-build support,
# it is only meant to test the copy of socket files
dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info"
package_folder = script.site_packages / "fspkg"
to_copy = data.packages.joinpath("FSPkg")
to_install = tmpdir.joinpath("src")
shutil.copytree(to_copy, to_install)
# Socket file, should be ignored.
socket_file_path = os.path.join(to_install, "example")
make_socket_file(socket_file_path)
result = script.pip(
"install",
"--use-deprecated=out-of-tree-build",
"--verbose",
to_install,
allow_stderr_warning=True, # because of the out-of-tree deprecation warning
)
result.did_create(package_folder)
result.did_create(dist_info_folder)
assert str(socket_file_path) in result.stderr
def test_install_from_local_directory_with_no_setup_py(
script: PipTestEnvironment, data: TestData
) -> None:

View File

@ -1,38 +1,10 @@
"""Helpers for filesystem-dependent tests.
"""
import os
import socket
import subprocess
import sys
from functools import partial
from itertools import chain
from typing import Iterator, List, Set
from .path import Path
def make_socket_file(path: str) -> None:
# Socket paths are limited to 108 characters (sometimes less) so we
# chdir before creating it and use a relative path name.
cwd = os.getcwd()
os.chdir(os.path.dirname(path))
try:
sock = socket.socket(socket.AF_UNIX)
sock.bind(os.path.basename(path))
finally:
os.chdir(cwd)
def make_unreadable_file(path: str) -> None:
Path(path).touch()
os.chmod(path, 0o000)
if sys.platform == "win32":
username = os.getlogin()
# Remove "Read Data/List Directory" permission for current user, but
# leave everything else.
args = ["icacls", path, "/deny", username + ":(RD)"]
subprocess.check_call(args)
def get_filelist(base: str) -> Set[str]:
def join(dirpath: str, dirnames: List[str], filenames: List[str]) -> Iterator[str]:

View File

@ -11,11 +11,10 @@ from pip._internal.exceptions import HashMismatch
from pip._internal.models.link import Link
from pip._internal.network.download import Downloader
from pip._internal.network.session import PipSession
from pip._internal.operations.prepare import _copy_source_tree, unpack_url
from pip._internal.operations.prepare import unpack_url
from pip._internal.utils.hashes import Hashes
from pip._internal.utils.urls import path_to_url
from tests.lib import TestData
from tests.lib.filesystem import get_filelist, make_socket_file, make_unreadable_file
from tests.lib.path import Path
from tests.lib.requests_mocks import MockResponse
@ -99,80 +98,6 @@ def clean_project(tmpdir_factory: pytest.TempdirFactory, data: TestData) -> Path
return new_project_dir
def test_copy_source_tree(clean_project: Path, tmpdir: Path) -> None:
target = tmpdir.joinpath("target")
expected_files = get_filelist(clean_project)
assert len(expected_files) == 3
_copy_source_tree(clean_project, target)
copied_files = get_filelist(target)
assert expected_files == copied_files
@pytest.mark.skipif("sys.platform == 'win32'")
def test_copy_source_tree_with_socket(
clean_project: Path, tmpdir: Path, caplog: pytest.LogCaptureFixture
) -> None:
target = tmpdir.joinpath("target")
expected_files = get_filelist(clean_project)
socket_path = str(clean_project.joinpath("aaa"))
make_socket_file(socket_path)
_copy_source_tree(clean_project, target)
copied_files = get_filelist(target)
assert expected_files == copied_files
# Warning should have been logged.
assert len(caplog.records) == 1
record = caplog.records[0]
assert record.levelname == "WARNING"
assert socket_path in record.message
@pytest.mark.skipif("sys.platform == 'win32'")
def test_copy_source_tree_with_socket_fails_with_no_socket_error(
clean_project: Path, tmpdir: Path
) -> None:
target = tmpdir.joinpath("target")
expected_files = get_filelist(clean_project)
make_socket_file(clean_project.joinpath("aaa"))
unreadable_file = clean_project.joinpath("bbb")
make_unreadable_file(unreadable_file)
with pytest.raises(shutil.Error) as e:
_copy_source_tree(clean_project, target)
errored_files = [err[0] for err in e.value.args[0]]
assert len(errored_files) == 1
assert unreadable_file in errored_files
copied_files = get_filelist(target)
# All files without errors should have been copied.
assert expected_files == copied_files
def test_copy_source_tree_with_unreadable_dir_fails(
clean_project: Path, tmpdir: Path
) -> None:
target = tmpdir.joinpath("target")
expected_files = get_filelist(clean_project)
unreadable_file = clean_project.joinpath("bbb")
make_unreadable_file(unreadable_file)
with pytest.raises(shutil.Error) as e:
_copy_source_tree(clean_project, target)
errored_files = [err[0] for err in e.value.args[0]]
assert len(errored_files) == 1
assert unreadable_file in errored_files
copied_files = get_filelist(target)
# All files without errors should have been copied.
assert expected_files == copied_files
class Test_unpack_url:
def prep(self, tmpdir: Path, data: TestData) -> None:
self.build_dir = tmpdir.joinpath("build")
@ -208,50 +133,3 @@ class Test_unpack_url:
hashes=Hashes({"md5": ["bogus"]}),
verbosity=0,
)
def test_unpack_url_thats_a_dir(self, tmpdir: Path, data: TestData) -> None:
self.prep(tmpdir, data)
dist_path = data.packages.joinpath("FSPkg")
dist_url = Link(path_to_url(dist_path))
unpack_url(
dist_url,
self.build_dir,
download=self.no_download,
download_dir=self.download_dir,
verbosity=0,
)
assert os.path.isdir(os.path.join(self.build_dir, "fspkg"))
@pytest.mark.parametrize("exclude_dir", [".nox", ".tox"])
def test_unpack_url_excludes_expected_dirs(tmpdir: Path, exclude_dir: str) -> None:
src_dir = tmpdir / "src"
dst_dir = tmpdir / "dst"
src_included_file = src_dir.joinpath("file.txt")
src_excluded_dir = src_dir.joinpath(exclude_dir)
src_excluded_file = src_dir.joinpath(exclude_dir, "file.txt")
src_included_dir = src_dir.joinpath("subdir", exclude_dir)
# set up source directory
src_excluded_dir.mkdir(parents=True)
src_included_dir.mkdir(parents=True)
src_included_file.touch()
src_excluded_file.touch()
dst_included_file = dst_dir.joinpath("file.txt")
dst_excluded_dir = dst_dir.joinpath(exclude_dir)
dst_excluded_file = dst_dir.joinpath(exclude_dir, "file.txt")
dst_included_dir = dst_dir.joinpath("subdir", exclude_dir)
src_link = Link(path_to_url(src_dir))
unpack_url(
src_link,
dst_dir,
Mock(side_effect=AssertionError),
download_dir=None,
verbosity=0,
)
assert not os.path.isdir(dst_excluded_dir)
assert not os.path.isfile(dst_excluded_file)
assert os.path.isfile(dst_included_file)
assert os.path.isdir(dst_included_dir)

View File

@ -99,7 +99,6 @@ class TestRequirementSet:
use_user_site=False,
lazy_wheel=False,
verbosity=0,
in_tree_build=False,
)
yield Resolver(
preparer=preparer,

View File

@ -1,11 +1,5 @@
import os
import shutil
from typing import Callable, Type
import pytest
from pip._internal.utils.filesystem import copy2_fixed, is_socket
from tests.lib.filesystem import make_socket_file, make_unreadable_file
from tests.lib.path import Path
@ -25,44 +19,3 @@ def make_broken_symlink(path: str) -> None:
def make_dir(path: str) -> None:
os.mkdir(path)
skip_on_windows = pytest.mark.skipif("sys.platform == 'win32'")
@skip_on_windows
@pytest.mark.parametrize(
"create,result",
[
(make_socket_file, True),
(make_file, False),
(make_valid_symlink, False),
(make_broken_symlink, False),
(make_dir, False),
],
)
def test_is_socket(create: Callable[[str], None], result: bool, tmpdir: Path) -> None:
target = tmpdir.joinpath("target")
create(target)
assert os.path.lexists(target)
assert is_socket(target) == result
@pytest.mark.parametrize(
"create,error_type",
[
pytest.param(make_socket_file, shutil.SpecialFileError, marks=skip_on_windows),
(make_unreadable_file, OSError),
],
)
def test_copy2_fixed_raises_appropriate_errors(
create: Callable[[str], None], error_type: Type[Exception], tmpdir: Path
) -> None:
src = tmpdir.joinpath("src")
create(src)
dest = tmpdir.joinpath("dest")
with pytest.raises(error_type):
copy2_fixed(src, dest)
assert not dest.exists()