McSinyx
/
pip
mirror of https://github.com/pypa/pip
1
1
Fork 0
Code Issues Releases Activity

Move WheelBuilder and friends to a dedicated module

This commit is contained in:
Pradyun Gedam 2019-11-03 18:37:30 +05:30
parent e79fa0e8a2
commit da9a432576
No known key found for this signature in database
GPG Key ID: DA17C4B29CB32E4B
5 changed files with 552 additions and 527 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/pip/_internal/commands/install.py
View File

@ -42,7 +42,7 @@ from pip._internal.utils.misc import (
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.utils.virtualenv import virtualenv_no_global
from pip._internal.wheel import WheelBuilder
from pip._internal.wheel_builder import WheelBuilder
if MYPY_CHECK_RUNNING:
from optparse import Values
@ -50,7 +50,7 @@ if MYPY_CHECK_RUNNING:
from pip._internal.models.format_control import FormatControl
from pip._internal.req.req_install import InstallRequirement
from pip._internal.wheel import BinaryAllowedPredicate
from pip._internal.wheel_builder import BinaryAllowedPredicate
logger = logging.getLogger(__name__)

2
src/pip/_internal/commands/wheel.py
View File

@ -16,7 +16,7 @@ from pip._internal.req import RequirementSet
from pip._internal.req.req_tracker import RequirementTracker
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.wheel import WheelBuilder
from pip._internal.wheel_builder import WheelBuilder
if MYPY_CHECK_RUNNING:
from optparse import Values

515
src/pip/_internal/wheel.py
View File

@ -9,7 +9,6 @@ from __future__ import absolute_import
import collections
import compileall
import csv
import hashlib
import logging
import os.path
import re
@ -26,50 +25,25 @@ from pip._vendor.distlib.util import get_export_entry
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.six import StringIO
from pip._internal import pep425tags
from pip._internal.exceptions import (
InstallationError,
InvalidWheelFilename,
UnsupportedWheel,
)
from pip._internal.locations import get_major_minor_version
from pip._internal.models.link import Link
from pip._internal.utils.logging import indent_log
from pip._internal.utils.marker_files import has_delete_marker_file
from pip._internal.utils.misc import captured_stdout, ensure_dir, read_chunks
from pip._internal.utils.setuptools_build import (
make_setuptools_bdist_wheel_args,
make_setuptools_clean_args,
)
from pip._internal.utils.subprocess import (
LOG_DIVIDER,
call_subprocess,
format_command_args,
runner_with_spinner_message,
)
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.misc import captured_stdout, ensure_dir
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.utils.ui import open_spinner
from pip._internal.utils.unpacking import unpack_file
from pip._internal.utils.urls import path_to_url
from pip._internal.vcs import vcs
from pip._internal.wheel_builder import hash_file
if MYPY_CHECK_RUNNING:
from typing import (
Dict, List, Optional, Sequence, Mapping, Tuple, IO, Text, Any,
Iterable, Callable, Set, Pattern, Union,
Iterable, Callable, Set,
)
from pip._internal.req.req_install import InstallRequirement
from pip._internal.operations.prepare import (
RequirementPreparer
)
from pip._internal.cache import WheelCache
from pip._internal.pep425tags import Pep425Tag
InstalledCSVRow = Tuple[str, ...]
BinaryAllowedPredicate = Callable[[InstallRequirement], bool]
VERSION_COMPATIBLE = (1, 0)
@ -82,18 +56,6 @@ def normpath(src, p):
return os.path.relpath(src, p).replace(os.path.sep, '/')
def hash_file(path, blocksize=1 << 20):
# type: (str, int) -> Tuple[Any, int]
"""Return (hash, length) for path using hashlib.sha256()"""
h = hashlib.sha256()
length = 0
with open(path, 'rb') as f:
for block in read_chunks(f, size=blocksize):
length += len(block)
h.update(block)
return (h, length) # type: ignore
def rehash(path, blocksize=1 << 20):
# type: (str, int) -> Tuple[str, str]
"""Return (encoded_digest, length) for path using hashlib.sha256()"""
@ -116,14 +78,6 @@ def open_for_csv(name, mode):
return open(name, mode + bin, **nl)
def replace_python_tag(wheelname, new_tag):
# type: (str, str) -> str
"""Replace the Python tag in a wheel file name with a new value."""
parts = wheelname.split('-')
parts[-3] = new_tag
return '-'.join(parts)
def fix_script(path):
# type: (str) -> Optional[bool]
"""Replace #!python with #!/path/to/python
@ -775,466 +729,3 @@ class Wheel(object):
:param tags: the PEP 425 tags to check the wheel against.
"""
return not self.file_tags.isdisjoint(tags)
def _contains_egg_info(
s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)):
# type: (str, Pattern) -> bool
"""Determine whether the string looks like an egg_info.
:param s: The string to parse. E.g. foo-2.1
"""
return bool(_egg_info_re.search(s))
def should_build(
req, # type: InstallRequirement
need_wheel, # type: bool
check_binary_allowed, # type: BinaryAllowedPredicate
):
# type: (...) -> Optional[bool]
"""Return whether an InstallRequirement should be built into a wheel."""
if req.constraint:
# never build requirements that are merely constraints
return False
if req.is_wheel:
if need_wheel:
logger.info(
'Skipping %s, due to already being wheel.', req.name,
)
return False
if need_wheel:
# i.e. pip wheel, not pip install
return True
if req.editable or not req.source_dir:
return False
if not check_binary_allowed(req):
logger.info(
"Skipping wheel build for %s, due to binaries "
"being disabled for it.", req.name,
)
return False
return True
def should_cache(
req, # type: InstallRequirement
check_binary_allowed, # type: BinaryAllowedPredicate
):
# type: (...) -> Optional[bool]
"""
Return whether a built InstallRequirement can be stored in the persistent
wheel cache, assuming the wheel cache is available, and should_build()
has determined a wheel needs to be built.
"""
if not should_build(
req, need_wheel=False, check_binary_allowed=check_binary_allowed
):
# never cache if pip install (need_wheel=False) would not have built
# (editable mode, etc)
return False
if req.link and req.link.is_vcs:
# VCS checkout. Build wheel just for this run
# unless it points to an immutable commit hash in which
# case it can be cached.
assert not req.editable
assert req.source_dir
vcs_backend = vcs.get_backend_for_scheme(req.link.scheme)
assert vcs_backend
if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir):
return True
return False
link = req.link
base, ext = link.splitext()
if _contains_egg_info(base):
return True
# Otherwise, build the wheel just for this run using the ephemeral
# cache since we are either in the case of e.g. a local directory, or
# no cache directory is available to use.
return False
def format_command_result(
command_args, # type: List[str]
command_output, # type: Text
):
# type: (...) -> str
"""Format command information for logging."""
command_desc = format_command_args(command_args)
text = 'Command arguments: {}\n'.format(command_desc)
if not command_output:
text += 'Command output: None'
elif logger.getEffectiveLevel() > logging.DEBUG:
text += 'Command output: [use --verbose to show]'
else:
if not command_output.endswith('\n'):
command_output += '\n'
text += 'Command output:\n{}{}'.format(command_output, LOG_DIVIDER)
return text
def get_legacy_build_wheel_path(
names, # type: List[str]
temp_dir, # type: str
req, # type: InstallRequirement
command_args, # type: List[str]
command_output, # type: Text
):
# type: (...) -> Optional[str]
"""Return the path to the wheel in the temporary build directory."""
# Sort for determinism.
names = sorted(names)
if not names:
msg = (
'Legacy build of wheel for {!r} created no files.\n'
).format(req.name)
msg += format_command_result(command_args, command_output)
logger.warning(msg)
return None
if len(names) > 1:
msg = (
'Legacy build of wheel for {!r} created more than one file.\n'
'Filenames (choosing first): {}\n'
).format(req.name, names)
msg += format_command_result(command_args, command_output)
logger.warning(msg)
return os.path.join(temp_dir, names[0])
def _always_true(_):
# type: (Any) -> bool
return True
class WheelBuilder(object):
"""Build wheels from a RequirementSet."""
def __init__(
self,
preparer, # type: RequirementPreparer
wheel_cache, # type: WheelCache
build_options=None, # type: Optional[List[str]]
global_options=None, # type: Optional[List[str]]
check_binary_allowed=None, # type: Optional[BinaryAllowedPredicate]
no_clean=False, # type: bool
path_to_wheelnames=None, # type: Optional[Union[bytes, Text]]
):
# type: (...) -> None
if check_binary_allowed is None:
# Binaries allowed by default.
check_binary_allowed = _always_true
self.preparer = preparer
self.wheel_cache = wheel_cache
self._wheel_dir = preparer.wheel_download_dir
self.build_options = build_options or []
self.global_options = global_options or []
self.check_binary_allowed = check_binary_allowed
self.no_clean = no_clean
# path where to save built names of built wheels
self.path_to_wheelnames = path_to_wheelnames
# file names of built wheel names
self.wheel_filenames = [] # type: List[Union[bytes, Text]]
def _build_one(
self,
req, # type: InstallRequirement
output_dir, # type: str
python_tag=None, # type: Optional[str]
):
# type: (...) -> Optional[str]
"""Build one wheel.
:return: The filename of the built wheel, or None if the build failed.
"""
# Install build deps into temporary directory (PEP 518)
with req.build_env:
return self._build_one_inside_env(req, output_dir,
python_tag=python_tag)
def _build_one_inside_env(
self,
req, # type: InstallRequirement
output_dir, # type: str
python_tag=None, # type: Optional[str]
):
# type: (...) -> Optional[str]
with TempDirectory(kind="wheel") as temp_dir:
if req.use_pep517:
builder = self._build_one_pep517
else:
builder = self._build_one_legacy
wheel_path = builder(req, temp_dir.path, python_tag=python_tag)
if wheel_path is not None:
wheel_name = os.path.basename(wheel_path)
dest_path = os.path.join(output_dir, wheel_name)
try:
wheel_hash, length = hash_file(wheel_path)
shutil.move(wheel_path, dest_path)
logger.info('Created wheel for %s: '
'filename=%s size=%d sha256=%s',
req.name, wheel_name, length,
wheel_hash.hexdigest())
logger.info('Stored in directory: %s', output_dir)
return dest_path
except Exception:
pass
# Ignore return, we can't do anything else useful.
self._clean_one(req)
return None
def _build_one_pep517(
self,
req, # type: InstallRequirement
tempd, # type: str
python_tag=None, # type: Optional[str]
):
# type: (...) -> Optional[str]
"""Build one InstallRequirement using the PEP 517 build process.
Returns path to wheel if successfully built. Otherwise, returns None.
"""
assert req.metadata_directory is not None
if self.build_options:
# PEP 517 does not support --build-options
logger.error('Cannot build wheel for %s using PEP 517 when '
'--build-options is present' % (req.name,))
return None
try:
logger.debug('Destination directory: %s', tempd)
runner = runner_with_spinner_message(
'Building wheel for {} (PEP 517)'.format(req.name)
)
backend = req.pep517_backend
with backend.subprocess_runner(runner):
wheel_name = backend.build_wheel(
tempd,
metadata_directory=req.metadata_directory,
)
if python_tag:
# General PEP 517 backends don't necessarily support
# a "--python-tag" option, so we rename the wheel
# file directly.
new_name = replace_python_tag(wheel_name, python_tag)
os.rename(
os.path.join(tempd, wheel_name),
os.path.join(tempd, new_name)
)
# Reassign to simplify the return at the end of function
wheel_name = new_name
except Exception:
logger.error('Failed building wheel for %s', req.name)
return None
return os.path.join(tempd, wheel_name)
def _build_one_legacy(
self,
req, # type: InstallRequirement
tempd, # type: str
python_tag=None, # type: Optional[str]
):
# type: (...) -> Optional[str]
"""Build one InstallRequirement using the "legacy" build process.
Returns path to wheel if successfully built. Otherwise, returns None.
"""
wheel_args = make_setuptools_bdist_wheel_args(
req.setup_py_path,
global_options=self.global_options,
build_options=self.build_options,
destination_dir=tempd,
python_tag=python_tag,
)
spin_message = 'Building wheel for %s (setup.py)' % (req.name,)
with open_spinner(spin_message) as spinner:
logger.debug('Destination directory: %s', tempd)
try:
output = call_subprocess(
wheel_args,
cwd=req.unpacked_source_directory,
spinner=spinner,
)
except Exception:
spinner.finish("error")
logger.error('Failed building wheel for %s', req.name)
return None
names = os.listdir(tempd)
wheel_path = get_legacy_build_wheel_path(
names=names,
temp_dir=tempd,
req=req,
command_args=wheel_args,
command_output=output,
)
return wheel_path
def _clean_one(self, req):
# type: (InstallRequirement) -> bool
clean_args = make_setuptools_clean_args(
req.setup_py_path,
global_options=self.global_options,
)
logger.info('Running setup.py clean for %s', req.name)
try:
call_subprocess(clean_args, cwd=req.source_dir)
return True
except Exception:
logger.error('Failed cleaning build dir for %s', req.name)
return False
def build(
self,
requirements, # type: Iterable[InstallRequirement]
should_unpack=False # type: bool
):
# type: (...) -> List[InstallRequirement]
"""Build wheels.
:param should_unpack: If True, after building the wheel, unpack it
and replace the sdist with the unpacked version in preparation
for installation.
:return: The list of InstallRequirement that failed to build.
"""
# pip install uses should_unpack=True.
# pip install never provides a _wheel_dir.
# pip wheel uses should_unpack=False.
# pip wheel always provides a _wheel_dir (via the preparer).
assert (
(should_unpack and not self._wheel_dir) or
(not should_unpack and self._wheel_dir)
)
buildset = []
cache_available = bool(self.wheel_cache.cache_dir)
for req in requirements:
if not should_build(
req,
need_wheel=not should_unpack,
check_binary_allowed=self.check_binary_allowed,
):
continue
if (
cache_available and
should_cache(req, self.check_binary_allowed)
):
output_dir = self.wheel_cache.get_path_for_link(req.link)
else:
output_dir = self.wheel_cache.get_ephem_path_for_link(
req.link
)
buildset.append((req, output_dir))
if not buildset:
return []
# TODO by @pradyunsg
# Should break up this method into 2 separate methods.
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for (req, _) in buildset]),
)
python_tag = None
if should_unpack:
python_tag = pep425tags.implementation_tag
with indent_log():
build_success, build_failure = [], []
for req, output_dir in buildset:
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning(
"Building wheel for %s failed: %s",
req.name, e,
)
build_failure.append(req)
continue
wheel_file = self._build_one(
req, output_dir,
python_tag=python_tag,
)
if wheel_file:
if should_unpack:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if (
req.source_dir and
not has_delete_marker_file(req.source_dir)
):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.ensure_build_location(
self.preparer.build_dir
)
# Update the link for this.
req.link = Link(path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_file(req.link.file_path, req.source_dir)
else:
# copy from cache to target directory
try:
ensure_dir(self._wheel_dir)
shutil.copy(
os.path.join(output_dir, wheel_file),
self._wheel_dir,
)
except OSError as e:
logger.warning(
"Building wheel for %s failed: %s",
req.name, e,
)
build_failure.append(req)
continue
self.wheel_filenames.append(
os.path.relpath(wheel_file, output_dir)
)
build_success.append(req)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return a list of requirements that failed to build
return build_failure

530
src/pip/_internal/wheel_builder.py Normal file
View File

@ -0,0 +1,530 @@
"""Orchestrator for building wheels from InstallRequirements.
"""
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import logging
import os.path
import re
import shutil
from pip._internal import pep425tags
from pip._internal.models.link import Link
from pip._internal.utils.logging import indent_log
from pip._internal.utils.marker_files import has_delete_marker_file
from pip._internal.utils.misc import ensure_dir, read_chunks
from pip._internal.utils.setuptools_build import (
make_setuptools_bdist_wheel_args,
make_setuptools_clean_args,
)
from pip._internal.utils.subprocess import (
LOG_DIVIDER,
call_subprocess,
format_command_args,
runner_with_spinner_message,
)
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.utils.ui import open_spinner
from pip._internal.utils.unpacking import unpack_file
from pip._internal.utils.urls import path_to_url
from pip._internal.vcs import vcs
if MYPY_CHECK_RUNNING:
from typing import (
Any, Callable, Iterable, List, Optional, Pattern, Text, Tuple, Union,
)
from pip._internal.cache import WheelCache
from pip._internal.operations.prepare import (
RequirementPreparer
)
from pip._internal.req.req_install import InstallRequirement
BinaryAllowedPredicate = Callable[[InstallRequirement], bool]
logger = logging.getLogger(__name__)
def hash_file(path, blocksize=1 << 20):
# type: (str, int) -> Tuple[Any, int]
"""Return (hash, length) for path using hashlib.sha256()"""
h = hashlib.sha256()
length = 0
with open(path, 'rb') as f:
for block in read_chunks(f, size=blocksize):
length += len(block)
h.update(block)
return (h, length) # type: ignore
def replace_python_tag(wheelname, new_tag):
# type: (str, str) -> str
"""Replace the Python tag in a wheel file name with a new value."""
parts = wheelname.split('-')
parts[-3] = new_tag
return '-'.join(parts)
def _contains_egg_info(
s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)):
# type: (str, Pattern) -> bool
"""Determine whether the string looks like an egg_info.
:param s: The string to parse. E.g. foo-2.1
"""
return bool(_egg_info_re.search(s))
def should_build(
req, # type: InstallRequirement
need_wheel, # type: bool
check_binary_allowed, # type: BinaryAllowedPredicate
):
# type: (...) -> Optional[bool]
"""Return whether an InstallRequirement should be built into a wheel."""
if req.constraint:
# never build requirements that are merely constraints
return False
if req.is_wheel:
if need_wheel:
logger.info(
'Skipping %s, due to already being wheel.', req.name,
)
return False
if need_wheel:
# i.e. pip wheel, not pip install
return True
if req.editable or not req.source_dir:
return False
if not check_binary_allowed(req):
logger.info(
"Skipping wheel build for %s, due to binaries "
"being disabled for it.", req.name,
)
return False
return True
def should_cache(
req, # type: InstallRequirement
check_binary_allowed, # type: BinaryAllowedPredicate
):
# type: (...) -> Optional[bool]
"""
Return whether a built InstallRequirement can be stored in the persistent
wheel cache, assuming the wheel cache is available, and should_build()
has determined a wheel needs to be built.
"""
if not should_build(
req, need_wheel=False, check_binary_allowed=check_binary_allowed
):
# never cache if pip install (need_wheel=False) would not have built
# (editable mode, etc)
return False
if req.link and req.link.is_vcs:
# VCS checkout. Build wheel just for this run
# unless it points to an immutable commit hash in which
# case it can be cached.
assert not req.editable
assert req.source_dir
vcs_backend = vcs.get_backend_for_scheme(req.link.scheme)
assert vcs_backend
if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir):
return True
return False
link = req.link
base, ext = link.splitext()
if _contains_egg_info(base):
return True
# Otherwise, build the wheel just for this run using the ephemeral
# cache since we are either in the case of e.g. a local directory, or
# no cache directory is available to use.
return False
def format_command_result(
command_args, # type: List[str]
command_output, # type: Text
):
# type: (...) -> str
"""Format command information for logging."""
command_desc = format_command_args(command_args)
text = 'Command arguments: {}\n'.format(command_desc)
if not command_output:
text += 'Command output: None'
elif logger.getEffectiveLevel() > logging.DEBUG:
text += 'Command output: [use --verbose to show]'
else:
if not command_output.endswith('\n'):
command_output += '\n'
text += 'Command output:\n{}{}'.format(command_output, LOG_DIVIDER)
return text
def get_legacy_build_wheel_path(
names, # type: List[str]
temp_dir, # type: str
req, # type: InstallRequirement
command_args, # type: List[str]
command_output, # type: Text
):
# type: (...) -> Optional[str]
"""Return the path to the wheel in the temporary build directory."""
# Sort for determinism.
names = sorted(names)
if not names:
msg = (
'Legacy build of wheel for {!r} created no files.\n'
).format(req.name)
msg += format_command_result(command_args, command_output)
logger.warning(msg)
return None
if len(names) > 1:
msg = (
'Legacy build of wheel for {!r} created more than one file.\n'
'Filenames (choosing first): {}\n'
).format(req.name, names)
msg += format_command_result(command_args, command_output)
logger.warning(msg)
return os.path.join(temp_dir, names[0])
def _always_true(_):
# type: (Any) -> bool
return True
class WheelBuilder(object):
"""Build wheels from a RequirementSet."""
def __init__(
self,
preparer, # type: RequirementPreparer
wheel_cache, # type: WheelCache
build_options=None, # type: Optional[List[str]]
global_options=None, # type: Optional[List[str]]
check_binary_allowed=None, # type: Optional[BinaryAllowedPredicate]
no_clean=False, # type: bool
path_to_wheelnames=None, # type: Optional[Union[bytes, Text]]
):
# type: (...) -> None
if check_binary_allowed is None:
# Binaries allowed by default.
check_binary_allowed = _always_true
self.preparer = preparer
self.wheel_cache = wheel_cache
self._wheel_dir = preparer.wheel_download_dir
self.build_options = build_options or []
self.global_options = global_options or []
self.check_binary_allowed = check_binary_allowed
self.no_clean = no_clean
# path where to save built names of built wheels
self.path_to_wheelnames = path_to_wheelnames
# file names of built wheel names
self.wheel_filenames = [] # type: List[Union[bytes, Text]]
def _build_one(
self,
req, # type: InstallRequirement
output_dir, # type: str
python_tag=None, # type: Optional[str]
):
# type: (...) -> Optional[str]
"""Build one wheel.
:return: The filename of the built wheel, or None if the build failed.
"""
# Install build deps into temporary directory (PEP 518)
with req.build_env:
return self._build_one_inside_env(req, output_dir,
python_tag=python_tag)
def _build_one_inside_env(
self,
req, # type: InstallRequirement
output_dir, # type: str
python_tag=None, # type: Optional[str]
):
# type: (...) -> Optional[str]
with TempDirectory(kind="wheel") as temp_dir:
if req.use_pep517:
builder = self._build_one_pep517
else:
builder = self._build_one_legacy
wheel_path = builder(req, temp_dir.path, python_tag=python_tag)
if wheel_path is not None:
wheel_name = os.path.basename(wheel_path)
dest_path = os.path.join(output_dir, wheel_name)
try:
wheel_hash, length = hash_file(wheel_path)
shutil.move(wheel_path, dest_path)
logger.info('Created wheel for %s: '
'filename=%s size=%d sha256=%s',
req.name, wheel_name, length,
wheel_hash.hexdigest())
logger.info('Stored in directory: %s', output_dir)
return dest_path
except Exception:
pass
# Ignore return, we can't do anything else useful.
self._clean_one(req)
return None
def _build_one_pep517(
self,
req, # type: InstallRequirement
tempd, # type: str
python_tag=None, # type: Optional[str]
):
# type: (...) -> Optional[str]
"""Build one InstallRequirement using the PEP 517 build process.
Returns path to wheel if successfully built. Otherwise, returns None.
"""
assert req.metadata_directory is not None
if self.build_options:
# PEP 517 does not support --build-options
logger.error('Cannot build wheel for %s using PEP 517 when '
'--build-options is present' % (req.name,))
return None
try:
logger.debug('Destination directory: %s', tempd)
runner = runner_with_spinner_message(
'Building wheel for {} (PEP 517)'.format(req.name)
)
backend = req.pep517_backend
with backend.subprocess_runner(runner):
wheel_name = backend.build_wheel(
tempd,
metadata_directory=req.metadata_directory,
)
if python_tag:
# General PEP 517 backends don't necessarily support
# a "--python-tag" option, so we rename the wheel
# file directly.
new_name = replace_python_tag(wheel_name, python_tag)
os.rename(
os.path.join(tempd, wheel_name),
os.path.join(tempd, new_name)
)
# Reassign to simplify the return at the end of function
wheel_name = new_name
except Exception:
logger.error('Failed building wheel for %s', req.name)
return None
return os.path.join(tempd, wheel_name)
def _build_one_legacy(
self,
req, # type: InstallRequirement
tempd, # type: str
python_tag=None, # type: Optional[str]
):
# type: (...) -> Optional[str]
"""Build one InstallRequirement using the "legacy" build process.
Returns path to wheel if successfully built. Otherwise, returns None.
"""
wheel_args = make_setuptools_bdist_wheel_args(
req.setup_py_path,
global_options=self.global_options,
build_options=self.build_options,
destination_dir=tempd,
python_tag=python_tag,
)
spin_message = 'Building wheel for %s (setup.py)' % (req.name,)
with open_spinner(spin_message) as spinner:
logger.debug('Destination directory: %s', tempd)
try:
output = call_subprocess(
wheel_args,
cwd=req.unpacked_source_directory,
spinner=spinner,
)
except Exception:
spinner.finish("error")
logger.error('Failed building wheel for %s', req.name)
return None
names = os.listdir(tempd)
wheel_path = get_legacy_build_wheel_path(
names=names,
temp_dir=tempd,
req=req,
command_args=wheel_args,
command_output=output,
)
return wheel_path
def _clean_one(self, req):
# type: (InstallRequirement) -> bool
clean_args = make_setuptools_clean_args(
req.setup_py_path,
global_options=self.global_options,
)
logger.info('Running setup.py clean for %s', req.name)
try:
call_subprocess(clean_args, cwd=req.source_dir)
return True
except Exception:
logger.error('Failed cleaning build dir for %s', req.name)
return False
def build(
self,
requirements, # type: Iterable[InstallRequirement]
should_unpack=False # type: bool
):
# type: (...) -> List[InstallRequirement]
"""Build wheels.
:param should_unpack: If True, after building the wheel, unpack it
and replace the sdist with the unpacked version in preparation
for installation.
:return: The list of InstallRequirement that failed to build.
"""
# pip install uses should_unpack=True.
# pip install never provides a _wheel_dir.
# pip wheel uses should_unpack=False.
# pip wheel always provides a _wheel_dir (via the preparer).
assert (
(should_unpack and not self._wheel_dir) or
(not should_unpack and self._wheel_dir)
)
buildset = []
cache_available = bool(self.wheel_cache.cache_dir)
for req in requirements:
if not should_build(
req,
need_wheel=not should_unpack,
check_binary_allowed=self.check_binary_allowed,
):
continue
if (
cache_available and
should_cache(req, self.check_binary_allowed)
):
output_dir = self.wheel_cache.get_path_for_link(req.link)
else:
output_dir = self.wheel_cache.get_ephem_path_for_link(
req.link
)
buildset.append((req, output_dir))
if not buildset:
return []
# TODO by @pradyunsg
# Should break up this method into 2 separate methods.
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for (req, _) in buildset]),
)
python_tag = None
if should_unpack:
python_tag = pep425tags.implementation_tag
with indent_log():
build_success, build_failure = [], []
for req, output_dir in buildset:
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning(
"Building wheel for %s failed: %s",
req.name, e,
)
build_failure.append(req)
continue
wheel_file = self._build_one(
req, output_dir,
python_tag=python_tag,
)
if wheel_file:
if should_unpack:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if (
req.source_dir and
not has_delete_marker_file(req.source_dir)
):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.ensure_build_location(
self.preparer.build_dir
)
# Update the link for this.
req.link = Link(path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_file(req.link.file_path, req.source_dir)
else:
# copy from cache to target directory
try:
ensure_dir(self._wheel_dir)
shutil.copy(
os.path.join(output_dir, wheel_file),
self._wheel_dir,
)
except OSError as e:
logger.warning(
"Building wheel for %s failed: %s",
req.name, e,
)
build_failure.append(req)
continue
self.wheel_filenames.append(
os.path.relpath(wheel_file, output_dir)
)
build_success.append(req)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return a list of requirements that failed to build
return build_failure

28
tests/unit/test_wheel.py
View File

@ -8,6 +8,7 @@ import pytest
from mock import Mock, patch
from pip._vendor.packaging.requirements import Requirement
import pip._internal.wheel_builder
from pip._internal import pep425tags, wheel
from pip._internal.commands.wheel import WheelCommand
from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel
@ -58,7 +59,7 @@ class ReqMock:
],
)
def test_contains_egg_info(s, expected):
result = wheel._contains_egg_info(s)
result = pip._internal.wheel_builder._contains_egg_info(s)
assert result == expected
@ -128,7 +129,7 @@ def test_format_tag(file_tag, expected):
],
)
def test_should_build(req, need_wheel, disallow_binaries, expected):
should_build = wheel.should_build(
should_build = pip._internal.wheel_builder.should_build(
req,
need_wheel,
check_binary_allowed=lambda req: not disallow_binaries,
@ -157,8 +158,10 @@ def test_should_cache(
def check_binary_allowed(req):
return not disallow_binaries
should_cache = wheel.should_cache(req, check_binary_allowed)
if not wheel.should_build(
should_cache = pip._internal.wheel_builder.should_cache(
req, check_binary_allowed
)
if not pip._internal.wheel_builder.should_build(
req, need_wheel=False, check_binary_allowed=check_binary_allowed
):
# never cache if pip install (need_wheel=False) would not have built)
@ -183,7 +186,7 @@ def test_should_cache_git_sha(script, tmpdir):
def test_format_command_result__INFO(caplog):
caplog.set_level(logging.INFO)
actual = wheel.format_command_result(
actual = pip._internal.wheel_builder.format_command_result(
# Include an argument with a space to test argument quoting.
command_args=['arg1', 'second arg'],
command_output='output line 1\noutput line 2\n',
@ -202,7 +205,7 @@ def test_format_command_result__INFO(caplog):
])
def test_format_command_result__DEBUG(caplog, command_output):
caplog.set_level(logging.DEBUG)
actual = wheel.format_command_result(
actual = pip._internal.wheel_builder.format_command_result(
command_args=['arg1', 'arg2'],
command_output=command_output,
)
@ -218,7 +221,7 @@ def test_format_command_result__DEBUG(caplog, command_output):
@pytest.mark.parametrize('log_level', ['DEBUG', 'INFO'])
def test_format_command_result__empty_output(caplog, log_level):
caplog.set_level(log_level)
actual = wheel.format_command_result(
actual = pip._internal.wheel_builder.format_command_result(
command_args=['arg1', 'arg2'],
command_output='',
)
@ -230,7 +233,7 @@ def test_format_command_result__empty_output(caplog, log_level):
def call_get_legacy_build_wheel_path(caplog, names):
req = make_test_install_req()
wheel_path = wheel.get_legacy_build_wheel_path(
wheel_path = pip._internal.wheel_builder.get_legacy_build_wheel_path(
names=names,
temp_dir='/tmp/abcd',
req=req,
@ -416,8 +419,9 @@ def test_python_tag():
'simplewheel-1.0-py37-none-any.whl',
'simplewheel-2.0-1-py37-none-any.whl',
]
for name, new in zip(wheelnames, newnames):
assert wheel.replace_python_tag(name, 'py37') == new
for name, expected in zip(wheelnames, newnames):
result = pip._internal.wheel_builder.replace_python_tag(name, 'py37')
assert result == expected
def test_check_compatibility():
@ -745,7 +749,7 @@ class TestWheelBuilder(object):
with patch('pip._internal.wheel.WheelBuilder._build_one') \
as mock_build_one:
wheel_req = Mock(is_wheel=True, editable=False, constraint=False)
wb = wheel.WheelBuilder(
wb = pip._internal.wheel_builder.WheelBuilder(
preparer=Mock(),
wheel_cache=Mock(cache_dir=None),
)
@ -883,7 +887,7 @@ class TestWheelHashCalculators(object):
def test_hash_file(self, tmpdir):
self.prep(tmpdir)
h, length = wheel.hash_file(self.test_file)
h, length = pip._internal.wheel_builder.hash_file(self.test_file)
assert length == self.test_file_len
assert h.hexdigest() == self.test_file_hash