mirror of
https://github.com/pypa/pip
synced 2023-12-13 21:30:23 +01:00
Merge pull request #7523 from sbidoul/wheel-builder-disentangle-7-sbi
Further reduce responsibility of WheelBuilder
This commit is contained in:
commit
e8ded43035
|
@ -74,6 +74,10 @@ def build_wheels(
|
||||||
builder, # type: WheelBuilder
|
builder, # type: WheelBuilder
|
||||||
pep517_requirements, # type: List[InstallRequirement]
|
pep517_requirements, # type: List[InstallRequirement]
|
||||||
legacy_requirements, # type: List[InstallRequirement]
|
legacy_requirements, # type: List[InstallRequirement]
|
||||||
|
wheel_cache, # type: WheelCache
|
||||||
|
build_options, # type: List[str]
|
||||||
|
global_options, # type: List[str]
|
||||||
|
check_binary_allowed, # type: BinaryAllowedPredicate
|
||||||
):
|
):
|
||||||
# type: (...) -> List[InstallRequirement]
|
# type: (...) -> List[InstallRequirement]
|
||||||
"""
|
"""
|
||||||
|
@ -86,6 +90,10 @@ def build_wheels(
|
||||||
_, build_failures = builder.build(
|
_, build_failures = builder.build(
|
||||||
pep517_requirements,
|
pep517_requirements,
|
||||||
should_unpack=True,
|
should_unpack=True,
|
||||||
|
wheel_cache=wheel_cache,
|
||||||
|
build_options=build_options,
|
||||||
|
global_options=global_options,
|
||||||
|
check_binary_allowed=check_binary_allowed,
|
||||||
)
|
)
|
||||||
|
|
||||||
if should_build_legacy:
|
if should_build_legacy:
|
||||||
|
@ -95,6 +103,10 @@ def build_wheels(
|
||||||
builder.build(
|
builder.build(
|
||||||
legacy_requirements,
|
legacy_requirements,
|
||||||
should_unpack=True,
|
should_unpack=True,
|
||||||
|
wheel_cache=wheel_cache,
|
||||||
|
build_options=build_options,
|
||||||
|
global_options=global_options,
|
||||||
|
check_binary_allowed=check_binary_allowed,
|
||||||
)
|
)
|
||||||
|
|
||||||
return build_failures
|
return build_failures
|
||||||
|
@ -396,16 +408,15 @@ class InstallCommand(RequirementCommand):
|
||||||
else:
|
else:
|
||||||
legacy_requirements.append(req)
|
legacy_requirements.append(req)
|
||||||
|
|
||||||
wheel_builder = WheelBuilder(
|
wheel_builder = WheelBuilder(preparer)
|
||||||
preparer, wheel_cache,
|
|
||||||
build_options=[], global_options=[],
|
|
||||||
check_binary_allowed=check_binary_allowed,
|
|
||||||
)
|
|
||||||
|
|
||||||
build_failures = build_wheels(
|
build_failures = build_wheels(
|
||||||
builder=wheel_builder,
|
builder=wheel_builder,
|
||||||
pep517_requirements=pep517_requirements,
|
pep517_requirements=pep517_requirements,
|
||||||
legacy_requirements=legacy_requirements,
|
legacy_requirements=legacy_requirements,
|
||||||
|
wheel_cache=wheel_cache,
|
||||||
|
build_options=[],
|
||||||
|
global_options=[],
|
||||||
|
check_binary_allowed=check_binary_allowed,
|
||||||
)
|
)
|
||||||
|
|
||||||
# If we're using PEP 517, we cannot do a direct install
|
# If we're using PEP 517, we cannot do a direct install
|
||||||
|
|
|
@ -158,14 +158,13 @@ class WheelCommand(RequirementCommand):
|
||||||
resolver.resolve(requirement_set)
|
resolver.resolve(requirement_set)
|
||||||
|
|
||||||
# build wheels
|
# build wheels
|
||||||
wb = WheelBuilder(
|
wb = WheelBuilder(preparer)
|
||||||
preparer, wheel_cache,
|
|
||||||
build_options=options.build_options or [],
|
|
||||||
global_options=options.global_options or [],
|
|
||||||
)
|
|
||||||
build_successes, build_failures = wb.build(
|
build_successes, build_failures = wb.build(
|
||||||
requirement_set.requirements.values(),
|
requirement_set.requirements.values(),
|
||||||
should_unpack=False,
|
should_unpack=False,
|
||||||
|
wheel_cache=wheel_cache,
|
||||||
|
build_options=options.build_options or [],
|
||||||
|
global_options=options.global_options or [],
|
||||||
)
|
)
|
||||||
for req in build_successes:
|
for req in build_successes:
|
||||||
assert req.link and req.link.is_wheel
|
assert req.link and req.link.is_wheel
|
||||||
|
|
|
@ -160,117 +160,115 @@ def _always_true(_):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _build_one(
|
||||||
|
req, # type: InstallRequirement
|
||||||
|
output_dir, # type: str
|
||||||
|
build_options, # type: List[str]
|
||||||
|
global_options, # type: List[str]
|
||||||
|
):
|
||||||
|
# type: (...) -> Optional[str]
|
||||||
|
"""Build one wheel.
|
||||||
|
|
||||||
|
:return: The filename of the built wheel, or None if the build failed.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
ensure_dir(output_dir)
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning(
|
||||||
|
"Building wheel for %s failed: %s",
|
||||||
|
req.name, e,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Install build deps into temporary directory (PEP 518)
|
||||||
|
with req.build_env:
|
||||||
|
return _build_one_inside_env(
|
||||||
|
req, output_dir, build_options, global_options
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_one_inside_env(
|
||||||
|
req, # type: InstallRequirement
|
||||||
|
output_dir, # type: str
|
||||||
|
build_options, # type: List[str]
|
||||||
|
global_options, # type: List[str]
|
||||||
|
):
|
||||||
|
# type: (...) -> Optional[str]
|
||||||
|
with TempDirectory(kind="wheel") as temp_dir:
|
||||||
|
if req.use_pep517:
|
||||||
|
wheel_path = build_wheel_pep517(
|
||||||
|
name=req.name,
|
||||||
|
backend=req.pep517_backend,
|
||||||
|
metadata_directory=req.metadata_directory,
|
||||||
|
build_options=build_options,
|
||||||
|
tempd=temp_dir.path,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
wheel_path = build_wheel_legacy(
|
||||||
|
name=req.name,
|
||||||
|
setup_py_path=req.setup_py_path,
|
||||||
|
source_dir=req.unpacked_source_directory,
|
||||||
|
global_options=global_options,
|
||||||
|
build_options=build_options,
|
||||||
|
tempd=temp_dir.path,
|
||||||
|
)
|
||||||
|
|
||||||
|
if wheel_path is not None:
|
||||||
|
wheel_name = os.path.basename(wheel_path)
|
||||||
|
dest_path = os.path.join(output_dir, wheel_name)
|
||||||
|
try:
|
||||||
|
wheel_hash, length = hash_file(wheel_path)
|
||||||
|
shutil.move(wheel_path, dest_path)
|
||||||
|
logger.info('Created wheel for %s: '
|
||||||
|
'filename=%s size=%d sha256=%s',
|
||||||
|
req.name, wheel_name, length,
|
||||||
|
wheel_hash.hexdigest())
|
||||||
|
logger.info('Stored in directory: %s', output_dir)
|
||||||
|
return dest_path
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(
|
||||||
|
"Building wheel for %s failed: %s",
|
||||||
|
req.name, e,
|
||||||
|
)
|
||||||
|
# Ignore return, we can't do anything else useful.
|
||||||
|
_clean_one(req, global_options)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_one(req, global_options):
|
||||||
|
# type: (InstallRequirement, List[str]) -> bool
|
||||||
|
clean_args = make_setuptools_clean_args(
|
||||||
|
req.setup_py_path,
|
||||||
|
global_options=global_options,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info('Running setup.py clean for %s', req.name)
|
||||||
|
try:
|
||||||
|
call_subprocess(clean_args, cwd=req.source_dir)
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
logger.error('Failed cleaning build dir for %s', req.name)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
class WheelBuilder(object):
|
class WheelBuilder(object):
|
||||||
"""Build wheels from a RequirementSet."""
|
"""Build wheels from a RequirementSet."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
preparer, # type: RequirementPreparer
|
preparer, # type: RequirementPreparer
|
||||||
wheel_cache, # type: WheelCache
|
|
||||||
build_options=None, # type: Optional[List[str]]
|
|
||||||
global_options=None, # type: Optional[List[str]]
|
|
||||||
check_binary_allowed=None, # type: Optional[BinaryAllowedPredicate]
|
|
||||||
):
|
):
|
||||||
# type: (...) -> None
|
# type: (...) -> None
|
||||||
if check_binary_allowed is None:
|
|
||||||
# Binaries allowed by default.
|
|
||||||
check_binary_allowed = _always_true
|
|
||||||
|
|
||||||
self.preparer = preparer
|
self.preparer = preparer
|
||||||
self.wheel_cache = wheel_cache
|
|
||||||
|
|
||||||
self.build_options = build_options or []
|
|
||||||
self.global_options = global_options or []
|
|
||||||
self.check_binary_allowed = check_binary_allowed
|
|
||||||
|
|
||||||
def _build_one(
|
|
||||||
self,
|
|
||||||
req, # type: InstallRequirement
|
|
||||||
output_dir, # type: str
|
|
||||||
):
|
|
||||||
# type: (...) -> Optional[str]
|
|
||||||
"""Build one wheel.
|
|
||||||
|
|
||||||
:return: The filename of the built wheel, or None if the build failed.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
ensure_dir(output_dir)
|
|
||||||
except OSError as e:
|
|
||||||
logger.warning(
|
|
||||||
"Building wheel for %s failed: %s",
|
|
||||||
req.name, e,
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Install build deps into temporary directory (PEP 518)
|
|
||||||
with req.build_env:
|
|
||||||
return self._build_one_inside_env(req, output_dir)
|
|
||||||
|
|
||||||
def _build_one_inside_env(
|
|
||||||
self,
|
|
||||||
req, # type: InstallRequirement
|
|
||||||
output_dir, # type: str
|
|
||||||
):
|
|
||||||
# type: (...) -> Optional[str]
|
|
||||||
with TempDirectory(kind="wheel") as temp_dir:
|
|
||||||
if req.use_pep517:
|
|
||||||
wheel_path = build_wheel_pep517(
|
|
||||||
name=req.name,
|
|
||||||
backend=req.pep517_backend,
|
|
||||||
metadata_directory=req.metadata_directory,
|
|
||||||
build_options=self.build_options,
|
|
||||||
tempd=temp_dir.path,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
wheel_path = build_wheel_legacy(
|
|
||||||
name=req.name,
|
|
||||||
setup_py_path=req.setup_py_path,
|
|
||||||
source_dir=req.unpacked_source_directory,
|
|
||||||
global_options=self.global_options,
|
|
||||||
build_options=self.build_options,
|
|
||||||
tempd=temp_dir.path,
|
|
||||||
)
|
|
||||||
|
|
||||||
if wheel_path is not None:
|
|
||||||
wheel_name = os.path.basename(wheel_path)
|
|
||||||
dest_path = os.path.join(output_dir, wheel_name)
|
|
||||||
try:
|
|
||||||
wheel_hash, length = hash_file(wheel_path)
|
|
||||||
shutil.move(wheel_path, dest_path)
|
|
||||||
logger.info('Created wheel for %s: '
|
|
||||||
'filename=%s size=%d sha256=%s',
|
|
||||||
req.name, wheel_name, length,
|
|
||||||
wheel_hash.hexdigest())
|
|
||||||
logger.info('Stored in directory: %s', output_dir)
|
|
||||||
return dest_path
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(
|
|
||||||
"Building wheel for %s failed: %s",
|
|
||||||
req.name, e,
|
|
||||||
)
|
|
||||||
# Ignore return, we can't do anything else useful.
|
|
||||||
self._clean_one(req)
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _clean_one(self, req):
|
|
||||||
# type: (InstallRequirement) -> bool
|
|
||||||
clean_args = make_setuptools_clean_args(
|
|
||||||
req.setup_py_path,
|
|
||||||
global_options=self.global_options,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info('Running setup.py clean for %s', req.name)
|
|
||||||
try:
|
|
||||||
call_subprocess(clean_args, cwd=req.source_dir)
|
|
||||||
return True
|
|
||||||
except Exception:
|
|
||||||
logger.error('Failed cleaning build dir for %s', req.name)
|
|
||||||
return False
|
|
||||||
|
|
||||||
def build(
|
def build(
|
||||||
self,
|
self,
|
||||||
requirements, # type: Iterable[InstallRequirement]
|
requirements, # type: Iterable[InstallRequirement]
|
||||||
should_unpack, # type: bool
|
should_unpack, # type: bool
|
||||||
|
wheel_cache, # type: WheelCache
|
||||||
|
build_options, # type: List[str]
|
||||||
|
global_options, # type: List[str]
|
||||||
|
check_binary_allowed=None, # type: Optional[BinaryAllowedPredicate]
|
||||||
):
|
):
|
||||||
# type: (...) -> BuildResult
|
# type: (...) -> BuildResult
|
||||||
"""Build wheels.
|
"""Build wheels.
|
||||||
|
@ -281,10 +279,14 @@ class WheelBuilder(object):
|
||||||
:return: The list of InstallRequirement that succeeded to build and
|
:return: The list of InstallRequirement that succeeded to build and
|
||||||
the list of InstallRequirement that failed to build.
|
the list of InstallRequirement that failed to build.
|
||||||
"""
|
"""
|
||||||
|
if check_binary_allowed is None:
|
||||||
|
# Binaries allowed by default.
|
||||||
|
check_binary_allowed = _always_true
|
||||||
|
|
||||||
buildset = _collect_buildset(
|
buildset = _collect_buildset(
|
||||||
requirements,
|
requirements,
|
||||||
wheel_cache=self.wheel_cache,
|
wheel_cache=wheel_cache,
|
||||||
check_binary_allowed=self.check_binary_allowed,
|
check_binary_allowed=check_binary_allowed,
|
||||||
need_wheel=not should_unpack,
|
need_wheel=not should_unpack,
|
||||||
)
|
)
|
||||||
if not buildset:
|
if not buildset:
|
||||||
|
@ -302,7 +304,9 @@ class WheelBuilder(object):
|
||||||
with indent_log():
|
with indent_log():
|
||||||
build_successes, build_failures = [], []
|
build_successes, build_failures = [], []
|
||||||
for req, cache_dir in buildset:
|
for req, cache_dir in buildset:
|
||||||
wheel_file = self._build_one(req, cache_dir)
|
wheel_file = _build_one(
|
||||||
|
req, cache_dir, build_options, global_options
|
||||||
|
)
|
||||||
if wheel_file:
|
if wheel_file:
|
||||||
# Update the link for this.
|
# Update the link for this.
|
||||||
req.link = Link(path_to_url(wheel_file))
|
req.link = Link(path_to_url(wheel_file))
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import errno
|
import errno
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from mock import Mock, call, patch
|
from mock import Mock, patch
|
||||||
from pip._vendor.packaging.requirements import Requirement
|
from pip._vendor.packaging.requirements import Requirement
|
||||||
|
|
||||||
from pip._internal.commands.install import (
|
from pip._internal.commands.install import (
|
||||||
|
@ -24,8 +24,11 @@ class TestWheelCache:
|
||||||
"""
|
"""
|
||||||
Return: (mock_calls, return_value).
|
Return: (mock_calls, return_value).
|
||||||
"""
|
"""
|
||||||
|
built_reqs = []
|
||||||
|
|
||||||
def build(reqs, **kwargs):
|
def build(reqs, **kwargs):
|
||||||
# Fail the first requirement.
|
# Fail the first requirement.
|
||||||
|
built_reqs.append(reqs)
|
||||||
return ([], [reqs[0]])
|
return ([], [reqs[0]])
|
||||||
|
|
||||||
builder = Mock()
|
builder = Mock()
|
||||||
|
@ -35,24 +38,25 @@ class TestWheelCache:
|
||||||
builder=builder,
|
builder=builder,
|
||||||
pep517_requirements=pep517_requirements,
|
pep517_requirements=pep517_requirements,
|
||||||
legacy_requirements=legacy_requirements,
|
legacy_requirements=legacy_requirements,
|
||||||
|
wheel_cache=Mock(cache_dir=None),
|
||||||
|
build_options=[],
|
||||||
|
global_options=[],
|
||||||
|
check_binary_allowed=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
return (builder.build.mock_calls, build_failures)
|
return (built_reqs, build_failures)
|
||||||
|
|
||||||
@patch('pip._internal.commands.install.is_wheel_installed')
|
@patch('pip._internal.commands.install.is_wheel_installed')
|
||||||
def test_build_wheels__wheel_installed(self, is_wheel_installed):
|
def test_build_wheels__wheel_installed(self, is_wheel_installed):
|
||||||
is_wheel_installed.return_value = True
|
is_wheel_installed.return_value = True
|
||||||
|
|
||||||
mock_calls, build_failures = self.check_build_wheels(
|
built_reqs, build_failures = self.check_build_wheels(
|
||||||
pep517_requirements=['a', 'b'],
|
pep517_requirements=['a', 'b'],
|
||||||
legacy_requirements=['c', 'd'],
|
legacy_requirements=['c', 'd'],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Legacy requirements were built.
|
# Legacy requirements were built.
|
||||||
assert mock_calls == [
|
assert built_reqs == [['a', 'b'], ['c', 'd']]
|
||||||
call(['a', 'b'], should_unpack=True),
|
|
||||||
call(['c', 'd'], should_unpack=True),
|
|
||||||
]
|
|
||||||
|
|
||||||
# Legacy build failures are not included in the return value.
|
# Legacy build failures are not included in the return value.
|
||||||
assert build_failures == ['a']
|
assert build_failures == ['a']
|
||||||
|
@ -61,15 +65,13 @@ class TestWheelCache:
|
||||||
def test_build_wheels__wheel_not_installed(self, is_wheel_installed):
|
def test_build_wheels__wheel_not_installed(self, is_wheel_installed):
|
||||||
is_wheel_installed.return_value = False
|
is_wheel_installed.return_value = False
|
||||||
|
|
||||||
mock_calls, build_failures = self.check_build_wheels(
|
built_reqs, build_failures = self.check_build_wheels(
|
||||||
pep517_requirements=['a', 'b'],
|
pep517_requirements=['a', 'b'],
|
||||||
legacy_requirements=['c', 'd'],
|
legacy_requirements=['c', 'd'],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Legacy requirements were not built.
|
# Legacy requirements were not built.
|
||||||
assert mock_calls == [
|
assert built_reqs == [['a', 'b']]
|
||||||
call(['a', 'b'], should_unpack=True),
|
|
||||||
]
|
|
||||||
|
|
||||||
assert build_failures == ['a']
|
assert build_failures == ['a']
|
||||||
|
|
||||||
|
|
|
@ -186,15 +186,18 @@ def test_format_command_result__empty_output(caplog, log_level):
|
||||||
class TestWheelBuilder(object):
|
class TestWheelBuilder(object):
|
||||||
|
|
||||||
def test_skip_building_wheels(self, caplog):
|
def test_skip_building_wheels(self, caplog):
|
||||||
wb = wheel_builder.WheelBuilder(
|
wb = wheel_builder.WheelBuilder(preparer=Mock())
|
||||||
preparer=Mock(),
|
|
||||||
wheel_cache=Mock(cache_dir=None),
|
|
||||||
)
|
|
||||||
wb._build_one = mock_build_one = Mock()
|
wb._build_one = mock_build_one = Mock()
|
||||||
|
|
||||||
wheel_req = Mock(is_wheel=True, editable=False, constraint=False)
|
wheel_req = Mock(is_wheel=True, editable=False, constraint=False)
|
||||||
with caplog.at_level(logging.INFO):
|
with caplog.at_level(logging.INFO):
|
||||||
wb.build([wheel_req], should_unpack=False)
|
wb.build(
|
||||||
|
[wheel_req],
|
||||||
|
should_unpack=False,
|
||||||
|
wheel_cache=Mock(cache_dir=None),
|
||||||
|
build_options=[],
|
||||||
|
global_options=[],
|
||||||
|
)
|
||||||
|
|
||||||
assert "due to already being wheel" in caplog.text
|
assert "due to already being wheel" in caplog.text
|
||||||
assert mock_build_one.mock_calls == []
|
assert mock_build_one.mock_calls == []
|
||||||
|
|
Loading…
Reference in a new issue