2018-01-22 22:23:56 +01:00
|
|
|
"""Build Environment used for isolation during sdist building
|
|
|
|
"""
|
|
|
|
|
2021-03-05 10:29:21 +01:00
|
|
|
import contextlib
|
2018-04-24 11:14:23 +02:00
|
|
|
import logging
|
2018-01-22 22:23:56 +01:00
|
|
|
import os
|
2021-03-05 10:29:21 +01:00
|
|
|
import pathlib
|
2018-04-19 09:45:30 +02:00
|
|
|
import sys
|
2018-10-16 08:34:28 +02:00
|
|
|
import textwrap
|
2021-03-05 10:29:21 +01:00
|
|
|
import zipfile
|
2018-10-25 17:35:25 +02:00
|
|
|
from collections import OrderedDict
|
2018-01-22 22:23:56 +01:00
|
|
|
from sysconfig import get_paths
|
2021-02-19 13:56:59 +01:00
|
|
|
from types import TracebackType
|
2021-03-05 10:29:21 +01:00
|
|
|
from typing import TYPE_CHECKING, Iterable, Iterator, List, Optional, Set, Tuple, Type
|
2018-01-22 22:23:56 +01:00
|
|
|
|
2021-03-05 15:57:45 +01:00
|
|
|
from pip._vendor.certifi import where
|
2018-08-02 20:44:29 +02:00
|
|
|
from pip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet
|
|
|
|
|
2018-10-02 12:00:19 +02:00
|
|
|
from pip import __file__ as pip_location
|
2020-02-21 09:31:13 +01:00
|
|
|
from pip._internal.cli.spinners import open_spinner
|
2021-02-19 10:14:11 +01:00
|
|
|
from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
|
2019-09-30 07:45:24 +02:00
|
|
|
from pip._internal.utils.subprocess import call_subprocess
|
2020-01-31 04:04:37 +01:00
|
|
|
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
|
2018-01-22 22:23:56 +01:00
|
|
|
|
2021-01-02 00:16:16 +01:00
|
|
|
if TYPE_CHECKING:
|
2019-10-06 18:59:05 +02:00
|
|
|
from pip._internal.index.package_finder import PackageFinder
|
2018-12-17 12:13:23 +01:00
|
|
|
|
2018-04-24 11:14:23 +02:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
2018-10-25 17:35:25 +02:00
|
|
|
class _Prefix:
|
|
|
|
|
|
|
|
def __init__(self, path):
|
2018-12-17 12:13:23 +01:00
|
|
|
# type: (str) -> None
|
2018-10-25 17:35:25 +02:00
|
|
|
self.path = path
|
|
|
|
self.setup = False
|
|
|
|
self.bin_dir = get_paths(
|
|
|
|
'nt' if os.name == 'nt' else 'posix_prefix',
|
|
|
|
vars={'base': path, 'platbase': path}
|
|
|
|
)['scripts']
|
2021-02-19 10:14:11 +01:00
|
|
|
self.lib_dirs = get_prefixed_libs(path)
|
2018-10-25 17:35:25 +02:00
|
|
|
|
|
|
|
|
2021-03-05 10:29:21 +01:00
|
|
|
@contextlib.contextmanager
|
|
|
|
def _create_standalone_pip() -> Iterator[str]:
|
2021-03-05 22:01:12 +01:00
|
|
|
"""Create a "standalone pip" zip file.
|
|
|
|
|
2021-03-06 14:10:06 +01:00
|
|
|
The zip file's content is identical to the currently-running pip.
|
2021-03-05 22:01:12 +01:00
|
|
|
It will be used to install requirements into the build environment.
|
|
|
|
"""
|
2021-03-05 10:29:21 +01:00
|
|
|
source = pathlib.Path(pip_location).resolve().parent
|
2021-03-06 14:10:06 +01:00
|
|
|
|
|
|
|
# Return the current instance if it is already a zip file. This can happen
|
|
|
|
# if a PEP 517 requirement is an sdist itself.
|
|
|
|
if not source.is_dir() and source.parent.name == "__env_pip__.zip":
|
|
|
|
yield str(source)
|
|
|
|
return
|
|
|
|
|
2021-03-05 19:26:56 +01:00
|
|
|
with TempDirectory(kind="standalone-pip") as tmp_dir:
|
2021-03-06 14:10:06 +01:00
|
|
|
pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip")
|
2021-04-29 19:11:58 +02:00
|
|
|
with zipfile.ZipFile(pip_zip, "w", strict_timestamps=False) as zf:
|
2021-03-05 10:29:21 +01:00
|
|
|
for child in source.rglob("*"):
|
2021-03-06 13:55:02 +01:00
|
|
|
zf.write(child, child.relative_to(source.parent).as_posix())
|
2021-03-05 10:29:21 +01:00
|
|
|
yield os.path.join(pip_zip, "pip")
|
|
|
|
|
|
|
|
|
2020-12-24 22:23:07 +01:00
|
|
|
class BuildEnvironment:
|
2018-03-01 09:50:06 +01:00
|
|
|
"""Creates and manages an isolated environment to install build deps
|
2018-01-22 22:23:56 +01:00
|
|
|
"""
|
|
|
|
|
2018-04-19 09:45:30 +02:00
|
|
|
def __init__(self):
|
2018-12-17 12:13:23 +01:00
|
|
|
# type: () -> None
|
2020-01-31 04:10:47 +01:00
|
|
|
temp_dir = TempDirectory(
|
2020-01-31 04:08:57 +01:00
|
|
|
kind=tempdir_kinds.BUILD_ENV, globally_managed=True
|
|
|
|
)
|
2018-01-22 22:23:56 +01:00
|
|
|
|
2021-02-10 09:46:25 +01:00
|
|
|
self._prefixes = OrderedDict(
|
2020-01-31 04:10:47 +01:00
|
|
|
(name, _Prefix(os.path.join(temp_dir.path, name)))
|
2018-10-25 17:35:25 +02:00
|
|
|
for name in ('normal', 'overlay')
|
2021-02-10 09:46:25 +01:00
|
|
|
)
|
2018-10-25 17:35:25 +02:00
|
|
|
|
2018-12-17 12:13:23 +01:00
|
|
|
self._bin_dirs = [] # type: List[str]
|
|
|
|
self._lib_dirs = [] # type: List[str]
|
2018-10-25 17:35:25 +02:00
|
|
|
for prefix in reversed(list(self._prefixes.values())):
|
|
|
|
self._bin_dirs.append(prefix.bin_dir)
|
|
|
|
self._lib_dirs.extend(prefix.lib_dirs)
|
|
|
|
|
|
|
|
# Customize site to:
|
|
|
|
# - ensure .pth files are honored
|
|
|
|
# - prevent access to system site packages
|
|
|
|
system_sites = {
|
2021-02-19 10:14:11 +01:00
|
|
|
os.path.normcase(site) for site in (get_purelib(), get_platlib())
|
2018-10-25 17:35:25 +02:00
|
|
|
}
|
2020-01-31 04:10:47 +01:00
|
|
|
self._site_dir = os.path.join(temp_dir.path, 'site')
|
2018-10-25 17:35:25 +02:00
|
|
|
if not os.path.exists(self._site_dir):
|
|
|
|
os.mkdir(self._site_dir)
|
|
|
|
with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp:
|
2018-10-16 08:34:28 +02:00
|
|
|
fp.write(textwrap.dedent(
|
|
|
|
'''
|
2018-10-25 17:35:25 +02:00
|
|
|
import os, site, sys
|
|
|
|
|
|
|
|
# First, drop system-sites related paths.
|
|
|
|
original_sys_path = sys.path[:]
|
|
|
|
known_paths = set()
|
|
|
|
for path in {system_sites!r}:
|
|
|
|
site.addsitedir(path, known_paths=known_paths)
|
|
|
|
system_paths = set(
|
|
|
|
os.path.normcase(path)
|
|
|
|
for path in sys.path[len(original_sys_path):]
|
|
|
|
)
|
|
|
|
original_sys_path = [
|
|
|
|
path for path in original_sys_path
|
|
|
|
if os.path.normcase(path) not in system_paths
|
|
|
|
]
|
|
|
|
sys.path = original_sys_path
|
|
|
|
|
|
|
|
# Second, add lib directories.
|
|
|
|
# ensuring .pth file are processed.
|
|
|
|
for path in {lib_dirs!r}:
|
|
|
|
assert not path in sys.path
|
|
|
|
site.addsitedir(path)
|
2018-10-16 08:34:28 +02:00
|
|
|
'''
|
2018-10-25 17:35:25 +02:00
|
|
|
).format(system_sites=system_sites, lib_dirs=self._lib_dirs))
|
2018-10-16 08:34:28 +02:00
|
|
|
|
2018-10-25 17:35:25 +02:00
|
|
|
def __enter__(self):
|
2020-05-21 23:30:55 +02:00
|
|
|
# type: () -> None
|
2018-10-25 17:35:25 +02:00
|
|
|
self._save_env = {
|
|
|
|
name: os.environ.get(name, None)
|
|
|
|
for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH')
|
|
|
|
}
|
|
|
|
|
|
|
|
path = self._bin_dirs[:]
|
|
|
|
old_path = self._save_env['PATH']
|
|
|
|
if old_path:
|
|
|
|
path.extend(old_path.split(os.pathsep))
|
|
|
|
|
|
|
|
pythonpath = [self._site_dir]
|
|
|
|
|
|
|
|
os.environ.update({
|
|
|
|
'PATH': os.pathsep.join(path),
|
|
|
|
'PYTHONNOUSERSITE': '1',
|
|
|
|
'PYTHONPATH': os.pathsep.join(pythonpath),
|
|
|
|
})
|
2018-01-22 22:23:56 +01:00
|
|
|
|
2020-05-21 23:30:55 +02:00
|
|
|
def __exit__(
|
|
|
|
self,
|
|
|
|
exc_type, # type: Optional[Type[BaseException]]
|
|
|
|
exc_val, # type: Optional[BaseException]
|
|
|
|
exc_tb # type: Optional[TracebackType]
|
|
|
|
):
|
|
|
|
# type: (...) -> None
|
2018-10-25 17:35:25 +02:00
|
|
|
for varname, old_value in self._save_env.items():
|
2018-03-02 21:56:14 +01:00
|
|
|
if old_value is None:
|
|
|
|
os.environ.pop(varname, None)
|
|
|
|
else:
|
|
|
|
os.environ[varname] = old_value
|
|
|
|
|
2018-10-25 17:35:25 +02:00
|
|
|
def check_requirements(self, reqs):
|
2018-12-17 12:13:23 +01:00
|
|
|
# type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
|
2018-10-25 17:35:25 +02:00
|
|
|
"""Return 2 sets:
|
|
|
|
- conflicting requirements: set of (installed, wanted) reqs tuples
|
|
|
|
- missing requirements: set of reqs
|
2018-08-02 20:44:29 +02:00
|
|
|
"""
|
2018-10-25 17:35:25 +02:00
|
|
|
missing = set()
|
|
|
|
conflicting = set()
|
|
|
|
if reqs:
|
|
|
|
ws = WorkingSet(self._lib_dirs)
|
2018-08-02 20:44:29 +02:00
|
|
|
for req in reqs:
|
|
|
|
try:
|
|
|
|
if ws.find(Requirement.parse(req)) is None:
|
2018-10-25 17:35:25 +02:00
|
|
|
missing.add(req)
|
|
|
|
except VersionConflict as e:
|
|
|
|
conflicting.add((str(e.args[0].as_requirement()),
|
|
|
|
str(e.args[1])))
|
|
|
|
return conflicting, missing
|
|
|
|
|
2018-12-17 12:13:23 +01:00
|
|
|
def install_requirements(
|
|
|
|
self,
|
|
|
|
finder, # type: PackageFinder
|
|
|
|
requirements, # type: Iterable[str]
|
|
|
|
prefix_as_string, # type: str
|
2020-05-23 05:45:25 +02:00
|
|
|
message # type: str
|
2018-12-17 12:13:23 +01:00
|
|
|
):
|
|
|
|
# type: (...) -> None
|
|
|
|
prefix = self._prefixes[prefix_as_string]
|
2018-10-25 17:35:25 +02:00
|
|
|
assert not prefix.setup
|
|
|
|
prefix.setup = True
|
|
|
|
if not requirements:
|
|
|
|
return
|
2021-04-27 08:27:42 +02:00
|
|
|
with contextlib.ExitStack() as ctx:
|
|
|
|
# TODO: Remove this block when dropping 3.6 support. Python 3.6
|
|
|
|
# lacks importlib.resources and pep517 has issues loading files in
|
|
|
|
# a zip, so we fallback to the "old" method by adding the current
|
|
|
|
# pip directory to the child process's sys.path.
|
|
|
|
if sys.version_info < (3, 7):
|
|
|
|
pip_runnable = os.path.dirname(pip_location)
|
|
|
|
else:
|
|
|
|
pip_runnable = ctx.enter_context(_create_standalone_pip())
|
2021-03-05 19:27:40 +01:00
|
|
|
self._install_requirements(
|
2021-04-27 08:27:42 +02:00
|
|
|
pip_runnable,
|
2021-03-05 10:29:21 +01:00
|
|
|
finder,
|
|
|
|
requirements,
|
|
|
|
prefix,
|
|
|
|
message,
|
|
|
|
)
|
2018-04-19 09:45:30 +02:00
|
|
|
|
2021-03-05 19:27:40 +01:00
|
|
|
@staticmethod
|
|
|
|
def _install_requirements(
|
2021-04-27 08:27:42 +02:00
|
|
|
pip_runnable: str,
|
2021-03-05 19:27:40 +01:00
|
|
|
finder: "PackageFinder",
|
|
|
|
requirements: Iterable[str],
|
|
|
|
prefix: _Prefix,
|
|
|
|
message: str,
|
|
|
|
) -> None:
|
|
|
|
args = [
|
2021-04-27 08:27:42 +02:00
|
|
|
sys.executable, pip_runnable, 'install',
|
2021-03-05 19:27:40 +01:00
|
|
|
'--ignore-installed', '--no-user', '--prefix', prefix.path,
|
2021-03-05 21:40:14 +01:00
|
|
|
'--no-warn-script-location',
|
2021-03-05 19:27:40 +01:00
|
|
|
] # type: List[str]
|
|
|
|
if logger.getEffectiveLevel() <= logging.DEBUG:
|
|
|
|
args.append('-v')
|
|
|
|
for format_control in ('no_binary', 'only_binary'):
|
|
|
|
formats = getattr(finder.format_control, format_control)
|
|
|
|
args.extend(('--' + format_control.replace('_', '-'),
|
|
|
|
','.join(sorted(formats or {':none:'}))))
|
|
|
|
|
|
|
|
index_urls = finder.index_urls
|
|
|
|
if index_urls:
|
|
|
|
args.extend(['-i', index_urls[0]])
|
|
|
|
for extra_index in index_urls[1:]:
|
|
|
|
args.extend(['--extra-index-url', extra_index])
|
|
|
|
else:
|
|
|
|
args.append('--no-index')
|
|
|
|
for link in finder.find_links:
|
|
|
|
args.extend(['--find-links', link])
|
|
|
|
|
|
|
|
for host in finder.trusted_hosts:
|
|
|
|
args.extend(['--trusted-host', host])
|
|
|
|
if finder.allow_all_prereleases:
|
|
|
|
args.append('--pre')
|
|
|
|
if finder.prefer_binary:
|
|
|
|
args.append('--prefer-binary')
|
|
|
|
args.append('--')
|
|
|
|
args.extend(requirements)
|
2021-03-06 13:55:02 +01:00
|
|
|
extra_environ = {"_PIP_STANDALONE_CERT": where()}
|
2021-03-05 19:27:40 +01:00
|
|
|
with open_spinner(message) as spinner:
|
2021-03-06 13:55:02 +01:00
|
|
|
call_subprocess(args, spinner=spinner, extra_environ=extra_environ)
|
2021-03-05 19:27:40 +01:00
|
|
|
|
2018-03-01 09:50:06 +01:00
|
|
|
|
|
|
|
class NoOpBuildEnvironment(BuildEnvironment):
|
|
|
|
"""A no-op drop-in replacement for BuildEnvironment
|
|
|
|
"""
|
|
|
|
|
2018-04-19 09:45:30 +02:00
|
|
|
def __init__(self):
|
2020-05-21 23:30:55 +02:00
|
|
|
# type: () -> None
|
2018-03-01 09:50:06 +01:00
|
|
|
pass
|
|
|
|
|
|
|
|
def __enter__(self):
|
2020-05-21 23:30:55 +02:00
|
|
|
# type: () -> None
|
2018-03-01 09:50:06 +01:00
|
|
|
pass
|
|
|
|
|
2020-05-21 23:30:55 +02:00
|
|
|
def __exit__(
|
|
|
|
self,
|
|
|
|
exc_type, # type: Optional[Type[BaseException]]
|
|
|
|
exc_val, # type: Optional[BaseException]
|
|
|
|
exc_tb # type: Optional[TracebackType]
|
|
|
|
):
|
|
|
|
# type: (...) -> None
|
2018-03-01 09:50:06 +01:00
|
|
|
pass
|
|
|
|
|
|
|
|
def cleanup(self):
|
2020-05-21 23:30:55 +02:00
|
|
|
# type: () -> None
|
2018-03-01 09:50:06 +01:00
|
|
|
pass
|
2018-04-19 09:45:30 +02:00
|
|
|
|
2020-05-21 23:30:55 +02:00
|
|
|
def install_requirements(
|
|
|
|
self,
|
|
|
|
finder, # type: PackageFinder
|
|
|
|
requirements, # type: Iterable[str]
|
|
|
|
prefix_as_string, # type: str
|
2020-05-23 05:45:25 +02:00
|
|
|
message # type: str
|
2020-05-21 23:30:55 +02:00
|
|
|
):
|
|
|
|
# type: (...) -> None
|
2018-04-19 09:45:30 +02:00
|
|
|
raise NotImplementedError()
|