2012-10-02 07:50:24 +02:00
|
|
|
"""
|
2013-04-02 07:44:46 +02:00
|
|
|
Support for installing and building the "wheel" binary package format.
|
2012-10-02 07:50:24 +02:00
|
|
|
"""
|
|
|
|
from __future__ import with_statement
|
|
|
|
|
|
|
|
import csv
|
|
|
|
import functools
|
|
|
|
import hashlib
|
2013-04-05 23:21:11 +02:00
|
|
|
import os
|
|
|
|
import pkg_resources
|
2013-04-02 07:44:46 +02:00
|
|
|
import re
|
2013-04-05 23:21:11 +02:00
|
|
|
import shutil
|
|
|
|
import sys
|
|
|
|
from base64 import urlsafe_b64encode
|
|
|
|
|
2012-11-15 01:53:22 +01:00
|
|
|
from pip.locations import distutils_scheme
|
2012-10-17 00:57:10 +02:00
|
|
|
from pip.log import logger
|
2013-07-01 05:22:55 +02:00
|
|
|
from pip import pep425tags
|
2013-04-05 23:21:11 +02:00
|
|
|
from pip.util import call_subprocess, normalize_path, make_path_relative
|
2012-10-02 07:50:24 +02:00
|
|
|
|
2013-04-02 07:44:46 +02:00
|
|
|
wheel_ext = '.whl'
|
2013-06-27 09:20:28 +02:00
|
|
|
# don't use pkg_resources.Requirement.parse, to avoid the override in distribute,
|
2013-07-27 06:51:35 +02:00
|
|
|
# that converts 'setuptools' to 'distribute'.
|
2013-07-06 07:20:09 +02:00
|
|
|
setuptools_requirement = list(pkg_resources.parse_requirements("setuptools>=0.8"))[0]
|
2012-10-02 07:50:24 +02:00
|
|
|
|
2013-06-14 07:04:05 +02:00
|
|
|
def wheel_setuptools_support():
|
2013-04-05 23:21:11 +02:00
|
|
|
"""
|
2013-06-27 09:20:28 +02:00
|
|
|
Return True if we have a setuptools that supports wheel.
|
2013-04-05 23:21:11 +02:00
|
|
|
"""
|
2013-06-27 09:20:28 +02:00
|
|
|
fulfilled = False
|
2013-03-16 19:40:02 +01:00
|
|
|
try:
|
2013-06-14 07:04:05 +02:00
|
|
|
installed_setuptools = pkg_resources.get_distribution('setuptools')
|
2013-06-27 09:20:28 +02:00
|
|
|
if installed_setuptools in setuptools_requirement:
|
|
|
|
fulfilled = True
|
2013-04-05 23:21:11 +02:00
|
|
|
except pkg_resources.DistributionNotFound:
|
2013-06-14 07:04:05 +02:00
|
|
|
pass
|
2013-06-27 09:20:28 +02:00
|
|
|
if not fulfilled:
|
|
|
|
logger.warn("%s is required for wheel installs." % setuptools_requirement)
|
|
|
|
return fulfilled
|
2013-04-02 07:44:46 +02:00
|
|
|
|
2012-10-02 07:50:24 +02:00
|
|
|
def rehash(path, algo='sha256', blocksize=1<<20):
|
|
|
|
"""Return (hash, length) for path using hashlib.new(algo)"""
|
|
|
|
h = hashlib.new(algo)
|
|
|
|
length = 0
|
|
|
|
with open(path) as f:
|
|
|
|
block = f.read(blocksize)
|
|
|
|
while block:
|
|
|
|
length += len(block)
|
|
|
|
h.update(block)
|
|
|
|
block = f.read(blocksize)
|
|
|
|
digest = 'sha256='+urlsafe_b64encode(h.digest()).decode('latin1').rstrip('=')
|
|
|
|
return (digest, length)
|
|
|
|
|
|
|
|
try:
|
|
|
|
unicode
|
|
|
|
def binary(s):
|
|
|
|
if isinstance(s, unicode):
|
|
|
|
return s.encode('ascii')
|
|
|
|
return s
|
|
|
|
except NameError:
|
|
|
|
def binary(s):
|
|
|
|
if isinstance(s, str):
|
|
|
|
return s.encode('ascii')
|
|
|
|
|
|
|
|
def open_for_csv(name, mode):
|
|
|
|
if sys.version_info[0] < 3:
|
|
|
|
nl = {}
|
|
|
|
bin = 'b'
|
|
|
|
else:
|
|
|
|
nl = { 'newline': '' }
|
|
|
|
bin = ''
|
|
|
|
return open(name, mode + bin, **nl)
|
|
|
|
|
|
|
|
def fix_script(path):
|
|
|
|
"""Replace #!python with #!/path/to/python
|
|
|
|
Return True if file was changed."""
|
|
|
|
# XXX RECORD hashes will need to be updated
|
|
|
|
if os.path.isfile(path):
|
|
|
|
script = open(path, 'rb')
|
|
|
|
try:
|
|
|
|
firstline = script.readline()
|
|
|
|
if not firstline.startswith(binary('#!python')):
|
|
|
|
return False
|
|
|
|
exename = sys.executable.encode(sys.getfilesystemencoding())
|
2012-10-02 16:48:54 +02:00
|
|
|
firstline = binary('#!') + exename + binary(os.linesep)
|
2012-10-02 07:50:24 +02:00
|
|
|
rest = script.read()
|
|
|
|
finally:
|
|
|
|
script.close()
|
|
|
|
script = open(path, 'wb')
|
|
|
|
try:
|
|
|
|
script.write(firstline)
|
|
|
|
script.write(rest)
|
|
|
|
finally:
|
|
|
|
script.close()
|
|
|
|
return True
|
2013-07-06 07:20:09 +02:00
|
|
|
|
2013-06-30 19:58:54 +02:00
|
|
|
dist_info_re = re.compile(r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
|
|
|
|
\.dist-info$""", re.VERBOSE)
|
|
|
|
|
|
|
|
def root_is_purelib(name, wheeldir):
|
|
|
|
"""
|
|
|
|
Return True if the extracted wheel in wheeldir should go into purelib.
|
|
|
|
"""
|
|
|
|
name_folded = name.replace("-", "_")
|
|
|
|
for item in os.listdir(wheeldir):
|
|
|
|
match = dist_info_re.match(item)
|
|
|
|
if match and match.group('name') == name_folded:
|
|
|
|
with open(os.path.join(wheeldir, item, 'WHEEL')) as wheel:
|
|
|
|
for line in wheel:
|
|
|
|
line = line.lower().rstrip()
|
|
|
|
if line == "root-is-purelib: true":
|
|
|
|
return True
|
|
|
|
return False
|
2012-10-02 07:50:24 +02:00
|
|
|
|
2012-11-15 01:53:22 +01:00
|
|
|
def move_wheel_files(name, req, wheeldir, user=False, home=None):
|
|
|
|
"""Install a wheel"""
|
2012-10-02 07:50:24 +02:00
|
|
|
|
2012-11-15 01:53:22 +01:00
|
|
|
scheme = distutils_scheme(name, user=user, home=home)
|
2013-07-06 07:20:09 +02:00
|
|
|
|
2013-06-30 19:58:54 +02:00
|
|
|
if root_is_purelib(name, wheeldir):
|
|
|
|
lib_dir = scheme['purelib']
|
|
|
|
else:
|
|
|
|
lib_dir = scheme['platlib']
|
2012-10-02 07:50:24 +02:00
|
|
|
|
|
|
|
info_dir = []
|
|
|
|
data_dirs = []
|
|
|
|
source = wheeldir.rstrip(os.path.sep) + os.path.sep
|
|
|
|
installed = {}
|
|
|
|
changed = set()
|
|
|
|
|
|
|
|
def normpath(src, p):
|
|
|
|
return make_path_relative(src, p).replace(os.path.sep, '/')
|
|
|
|
|
|
|
|
def record_installed(srcfile, destfile, modified=False):
|
|
|
|
"""Map archive RECORD paths to installation RECORD paths."""
|
|
|
|
oldpath = normpath(srcfile, wheeldir)
|
2013-06-30 19:58:54 +02:00
|
|
|
newpath = normpath(destfile, lib_dir)
|
2012-10-02 07:50:24 +02:00
|
|
|
installed[oldpath] = newpath
|
|
|
|
if modified:
|
|
|
|
changed.add(destfile)
|
|
|
|
|
|
|
|
def clobber(source, dest, is_base, fixer=None):
|
2013-05-24 04:42:03 +02:00
|
|
|
if not os.path.exists(dest): # common for the 'include' path
|
|
|
|
os.makedirs(dest)
|
|
|
|
|
2012-10-02 07:50:24 +02:00
|
|
|
for dir, subdirs, files in os.walk(source):
|
|
|
|
basedir = dir[len(source):].lstrip(os.path.sep)
|
|
|
|
if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
|
|
|
|
continue
|
|
|
|
for s in subdirs:
|
|
|
|
destsubdir = os.path.join(dest, basedir, s)
|
|
|
|
if is_base and basedir == '' and destsubdir.endswith('.data'):
|
|
|
|
data_dirs.append(s)
|
|
|
|
continue
|
|
|
|
elif (is_base
|
|
|
|
and s.endswith('.dist-info')
|
|
|
|
# is self.req.project_name case preserving?
|
|
|
|
and s.lower().startswith(req.project_name.replace('-', '_').lower())):
|
|
|
|
assert not info_dir, 'Multiple .dist-info directories'
|
|
|
|
info_dir.append(destsubdir)
|
|
|
|
if not os.path.exists(destsubdir):
|
|
|
|
os.makedirs(destsubdir)
|
|
|
|
for f in files:
|
|
|
|
srcfile = os.path.join(dir, f)
|
|
|
|
destfile = os.path.join(dest, basedir, f)
|
|
|
|
shutil.move(srcfile, destfile)
|
|
|
|
changed = False
|
|
|
|
if fixer:
|
|
|
|
changed = fixer(destfile)
|
|
|
|
record_installed(srcfile, destfile, changed)
|
|
|
|
|
2013-06-30 19:58:54 +02:00
|
|
|
clobber(source, lib_dir, True)
|
2012-10-02 07:50:24 +02:00
|
|
|
|
|
|
|
assert info_dir, "%s .dist-info directory not found" % req
|
|
|
|
|
|
|
|
for datadir in data_dirs:
|
|
|
|
fixer = None
|
|
|
|
for subdir in os.listdir(os.path.join(wheeldir, datadir)):
|
|
|
|
fixer = None
|
|
|
|
if subdir == 'scripts':
|
|
|
|
fixer = fix_script
|
|
|
|
source = os.path.join(wheeldir, datadir, subdir)
|
2012-11-15 01:53:22 +01:00
|
|
|
dest = scheme[subdir]
|
2012-10-02 07:50:24 +02:00
|
|
|
clobber(source, dest, False, fixer=fixer)
|
|
|
|
|
|
|
|
record = os.path.join(info_dir[0], 'RECORD')
|
|
|
|
temp_record = os.path.join(info_dir[0], 'RECORD.pip')
|
|
|
|
with open_for_csv(record, 'r') as record_in:
|
|
|
|
with open_for_csv(temp_record, 'w+') as record_out:
|
|
|
|
reader = csv.reader(record_in)
|
|
|
|
writer = csv.writer(record_out)
|
|
|
|
for row in reader:
|
|
|
|
row[0] = installed.pop(row[0], row[0])
|
|
|
|
if row[0] in changed:
|
|
|
|
row[1], row[2] = rehash(row[0])
|
|
|
|
writer.writerow(row)
|
|
|
|
for f in installed:
|
|
|
|
writer.writerow((installed[f], '', ''))
|
|
|
|
shutil.move(temp_record, record)
|
|
|
|
|
|
|
|
def _unique(fn):
|
|
|
|
@functools.wraps(fn)
|
|
|
|
def unique(*args, **kw):
|
|
|
|
seen = set()
|
|
|
|
for item in fn(*args, **kw):
|
|
|
|
if item not in seen:
|
|
|
|
seen.add(item)
|
|
|
|
yield item
|
|
|
|
return unique
|
|
|
|
|
2013-04-02 07:44:46 +02:00
|
|
|
# TODO: this goes somewhere besides the wheel module
|
2012-10-02 07:50:24 +02:00
|
|
|
@_unique
|
|
|
|
def uninstallation_paths(dist):
|
|
|
|
"""
|
|
|
|
Yield all the uninstallation paths for dist based on RECORD-without-.pyc
|
|
|
|
|
|
|
|
Yield paths to all the files in RECORD. For each .py file in RECORD, add
|
|
|
|
the .pyc in the same directory.
|
|
|
|
|
|
|
|
UninstallPathSet.add() takes care of the __pycache__ .pyc.
|
|
|
|
"""
|
|
|
|
from pip.req import FakeFile # circular import
|
|
|
|
r = csv.reader(FakeFile(dist.get_metadata_lines('RECORD')))
|
|
|
|
for row in r:
|
|
|
|
path = os.path.join(dist.location, row[0])
|
|
|
|
yield path
|
|
|
|
if path.endswith('.py'):
|
|
|
|
dn, fn = os.path.split(path)
|
|
|
|
base = fn[:-3]
|
|
|
|
path = os.path.join(dn, base+'.pyc')
|
|
|
|
yield path
|
2012-10-17 00:57:10 +02:00
|
|
|
|
|
|
|
|
2013-04-02 07:44:46 +02:00
|
|
|
class Wheel(object):
|
|
|
|
"""A wheel file"""
|
|
|
|
|
|
|
|
# TODO: maybe move the install code into this class
|
|
|
|
|
|
|
|
wheel_file_re = re.compile(
|
|
|
|
r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
|
|
|
|
((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
|
|
|
|
\.whl|\.dist-info)$""",
|
|
|
|
re.VERBOSE)
|
|
|
|
|
|
|
|
def __init__(self, filename):
|
|
|
|
wheel_info = self.wheel_file_re.match(filename)
|
|
|
|
self.filename = filename
|
|
|
|
self.name = wheel_info.group('name').replace('_', '-')
|
2013-08-22 08:30:15 +02:00
|
|
|
# we'll assume "_" means "-" due to wheel naming scheme
|
|
|
|
# (https://github.com/pypa/pip/issues/1150)
|
|
|
|
self.version = wheel_info.group('ver').replace('_', '-')
|
2013-04-02 07:44:46 +02:00
|
|
|
self.pyversions = wheel_info.group('pyver').split('.')
|
|
|
|
self.abis = wheel_info.group('abi').split('.')
|
|
|
|
self.plats = wheel_info.group('plat').split('.')
|
|
|
|
|
|
|
|
# All the tag combinations from this file
|
|
|
|
self.file_tags = set((x, y, z) for x in self.pyversions for y
|
|
|
|
in self.abis for z in self.plats)
|
|
|
|
|
2013-07-01 05:22:55 +02:00
|
|
|
def support_index_min(self, tags=None):
|
2013-04-02 07:44:46 +02:00
|
|
|
"""
|
|
|
|
Return the lowest index that a file_tag achieves in the supported_tags list
|
|
|
|
e.g. if there are 8 supported tags, and one of the file tags is first in the
|
|
|
|
list, then return 0.
|
|
|
|
"""
|
2013-07-01 05:22:55 +02:00
|
|
|
if tags is None: # for mock
|
|
|
|
tags = pep425tags.supported_tags
|
2013-06-30 20:56:43 +02:00
|
|
|
indexes = [tags.index(c) for c in self.file_tags if c in tags]
|
2013-04-02 07:44:46 +02:00
|
|
|
return min(indexes) if indexes else None
|
|
|
|
|
2013-07-01 05:22:55 +02:00
|
|
|
def supported(self, tags=None):
|
2013-04-02 07:44:46 +02:00
|
|
|
"""Is this wheel supported on this system?"""
|
2013-07-01 05:22:55 +02:00
|
|
|
if tags is None: # for mock
|
|
|
|
tags = pep425tags.supported_tags
|
2013-06-30 20:15:03 +02:00
|
|
|
return bool(set(tags).intersection(self.file_tags))
|
2013-04-02 07:44:46 +02:00
|
|
|
|
2012-10-17 00:57:10 +02:00
|
|
|
|
|
|
|
class WheelBuilder(object):
|
|
|
|
"""Build wheels from a RequirementSet."""
|
|
|
|
|
|
|
|
def __init__(self, requirement_set, finder, wheel_dir, build_options=[], global_options=[]):
|
|
|
|
self.requirement_set = requirement_set
|
|
|
|
self.finder = finder
|
|
|
|
self.wheel_dir = normalize_path(wheel_dir)
|
|
|
|
self.build_options = build_options
|
|
|
|
self.global_options = global_options
|
|
|
|
|
|
|
|
def _build_one(self, req):
|
|
|
|
"""Build one wheel."""
|
|
|
|
|
|
|
|
base_args = [
|
|
|
|
sys.executable, '-c',
|
|
|
|
"import setuptools;__file__=%r;"\
|
|
|
|
"exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))" % req.setup_py] + \
|
|
|
|
list(self.global_options)
|
|
|
|
|
|
|
|
logger.notify('Running setup.py bdist_wheel for %s' % req.name)
|
|
|
|
logger.notify('Destination directory: %s' % self.wheel_dir)
|
|
|
|
wheel_args = base_args + ['bdist_wheel', '-d', self.wheel_dir] + self.build_options
|
|
|
|
try:
|
|
|
|
call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False)
|
|
|
|
return True
|
|
|
|
except:
|
|
|
|
logger.error('Failed building wheel for %s' % req.name)
|
|
|
|
return False
|
|
|
|
|
|
|
|
def build(self):
|
|
|
|
"""Build wheels."""
|
|
|
|
|
2012-10-27 23:44:50 +02:00
|
|
|
#unpack and constructs req set
|
2012-10-17 00:57:10 +02:00
|
|
|
self.requirement_set.prepare_files(self.finder)
|
|
|
|
|
|
|
|
reqset = self.requirement_set.requirements.values()
|
|
|
|
|
|
|
|
#make the wheelhouse
|
|
|
|
if not os.path.exists(self.wheel_dir):
|
|
|
|
os.makedirs(self.wheel_dir)
|
|
|
|
|
|
|
|
#build the wheels
|
|
|
|
logger.notify('Building wheels for collected packages: %s' % ', '.join([req.name for req in reqset]))
|
|
|
|
logger.indent += 2
|
|
|
|
build_success, build_failure = [], []
|
|
|
|
for req in reqset:
|
2012-10-27 23:44:50 +02:00
|
|
|
if req.is_wheel:
|
2013-06-07 04:11:43 +02:00
|
|
|
logger.notify("Skipping building wheel: %s", req.url)
|
2012-10-27 23:44:50 +02:00
|
|
|
continue
|
2012-10-17 00:57:10 +02:00
|
|
|
if self._build_one(req):
|
|
|
|
build_success.append(req)
|
|
|
|
else:
|
|
|
|
build_failure.append(req)
|
|
|
|
logger.indent -= 2
|
|
|
|
|
|
|
|
#notify sucess/failure
|
|
|
|
if build_success:
|
|
|
|
logger.notify('Successfully built %s' % ' '.join([req.name for req in build_success]))
|
|
|
|
if build_failure:
|
|
|
|
logger.notify('Failed to build %s' % ' '.join([req.name for req in build_failure]))
|