experimental support for installing wheel archives

docs/news.txt

@@ -14,6 +14,9 @@ Beta and final releases planned for the end of 2012.
 develop (unreleased)
 --------------------
 
+* Pip now has experimental "wheel" support.  Thanks Daniel Holth, Paul Moore,
+  and Michele Lacchia.
+
 * Added check in ``install --download`` to prevent re-downloading if the target
   file already exists. Thanks Andrey Bulgakov.
 

docs/usage.txt

@@ -154,6 +154,24 @@ To get info about an installed package, including its location and included
 files, run ``pip show ProjectName``.
 
 
+Wheel support
+-------------
+
+Pip has experimental support for installing "wheel" archives, which requires distribute (not setuptools) and markerlib.
+
+To install directly from a wheel archive::
+
+    $ pip install /path/to/SomeWheel[...].whl
+
+To have pip find wheels on pypi indexes or find-links::
+
+    $ pip install --use-wheel SomeWheel
+
+Pip currently only supports finding wheels based on the python version tag, not implementation, abi or platform tags.
+
+For more information, see the `wheel documentation <http://wheel.readthedocs.org/en/latest/index.html>`_
+
+
 Bundles
 -------
 

pip/backwardcompat.py

@@ -110,3 +110,24 @@ def home_lib(home):
     else:
         lib = os.path.join('lib', 'python')
     return os.path.join(home, lib)
+try:
+    import sysconfig
+except: # pragma nocover
+    from distutils import sysconfig
+
+import pip.locations
+get_path_locations = {'purelib':pip.locations.site_packages,
+                      'platlib':pip.locations.site_packages,
+                      'scripts':pip.locations.bin_py,
+                      'data':sys.prefix}
+try:
+    sysconfig.get_path
+    def get_path(path):
+        try:
+            return get_path_locations[path]
+        except KeyError:
+            return sysconfig.get_path(path)
+except AttributeError: # Python < 2.7
+    from pip.locations import site_packages, bin_py
+    def get_path(path):
+        return get_path_locations[path]

pip/commands/install.py

@@ -20,6 +20,10 @@ class InstallCommand(Command):
 
     def __init__(self):
         super(InstallCommand, self).__init__()
+        self.parser.add_option('--use-wheel',
+            dest='use_wheel',
+            action='store_true',
+            help='Find wheel archives when searching index and find-links')
         self.parser.add_option(
             '-e', '--editable',
             dest='editables',
@@ -181,7 +185,8 @@ class InstallCommand(Command):
         return PackageFinder(find_links=options.find_links,
                              index_urls=index_urls,
                              use_mirrors=options.use_mirrors,
-                             mirrors=options.mirrors)
+                             mirrors=options.mirrors,
+                             use_wheel=options.use_wheel)
 
     def run(self, options, args):
         if options.download_dir:
@@ -214,6 +219,7 @@ class InstallCommand(Command):
             src_dir=options.src_dir,
             download_dir=options.download_dir,
             download_cache=options.download_cache,
+            use_wheel=options.use_wheel,
             upgrade=options.upgrade,
             as_egg=options.as_egg,
             ignore_installed=options.ignore_installed,

pip/commands/zip.py

@@ -134,7 +134,8 @@ class ZipCommand(Command):
             ## FIXME: this should be undoable:
             zip = zipfile.ZipFile(zip_filename)
             to_save = []
-            for name in zip.namelist():
+            for info in zip.infolist():
+                name = info.filename
                 if name.startswith(module_name + os.path.sep):
                     content = zip.read(name)
                     dest = os.path.join(package_path, name)

pip/download.py

@@ -280,7 +280,8 @@ def geturl(urllib2_resp):
 
 def is_archive_file(name):
     """Return True if `name` is a considered as an archive file."""
-    archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle')
+    archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle',
+               '.whl')
     ext = splitext(name)[1].lower()
     if ext in archives:
         return True

pip/index.py

@@ -40,7 +40,8 @@ class PackageFinder(object):
     """
 
     def __init__(self, find_links, index_urls,
-            use_mirrors=False, mirrors=None, main_mirror_url=None):
+            use_mirrors=False, mirrors=None, main_mirror_url=None,
+            use_wheel=False):
         self.find_links = find_links
         self.index_urls = index_urls
         self.dependency_links = []
@@ -52,6 +53,7 @@ class PackageFinder(object):
             logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls))
         else:
             self.mirror_urls = []
+        self.use_wheel = use_wheel
 
     def add_dependency_links(self, links):
         ## FIXME: this shouldn't be global list this, it should only
@@ -261,6 +263,11 @@ class PackageFinder(object):
     _egg_fragment_re = re.compile(r'#egg=([^&]*)')
     _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I)
     _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$')
+    _wheel_info_re = re.compile(
+                r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
+                ((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
+                \.whl|\.dist-info)$""",
+                re.VERBOSE)
 
     def _sort_links(self, links):
         "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates"
@@ -279,6 +286,12 @@ class PackageFinder(object):
         for link in self._sort_links(links):
             for v in self._link_package_versions(link, search_name):
                 yield v
+        
+    def _known_extensions(self):
+        extensions = ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip') 
+        if self.use_wheel:
+            return extensions + ('.whl',)
+        return extensions
 
     def _link_package_versions(self, link, search_name):
         """
@@ -288,6 +301,7 @@ class PackageFinder(object):
 
         Meant to be overridden by subclasses, not called by clients.
         """
+        version = None
         if link.egg_fragment:
             egg_info = link.egg_fragment
         else:
@@ -301,7 +315,7 @@ class PackageFinder(object):
                 # Special double-extension case:
                 egg_info = egg_info[:-4]
                 ext = '.tar' + ext
-            if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'):
+            if ext not in self._known_extensions():
                 if link not in self.logged_links:
                     logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext))
                     self.logged_links.add(link)
@@ -311,7 +325,23 @@ class PackageFinder(object):
                     logger.debug('Skipping link %s; macosx10 one' % (link))
                     self.logged_links.add(link)
                 return []
-        version = self._egg_info_matches(egg_info, search_name, link)
+            if ext == '.whl':
+                wheel_info = self._wheel_info_re.match(link.filename)
+                if wheel_info.group('name').replace('_', '-').lower() == search_name.lower():
+                    version = wheel_info.group('ver')
+                    nodot = sys.version[:3].replace('.', '')
+                    pyversions = wheel_info.group('pyver').split('.')
+                    ok = False
+                    for pv in pyversions:
+                        # TODO: Doesn't check Python implementation
+                        if nodot.startswith(pv[2:]):
+                            ok = True
+                            break
+                    if not ok:
+                        logger.debug('Skipping %s because Python version is incorrect' % link)
+                        return []    
+        if not version:
+            version = self._egg_info_matches(egg_info, search_name, link)
         if version is None:
             logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name))
             return []

pip/req.py

@@ -29,6 +29,8 @@ from pip.download import (get_file_content, is_url, url_to_path,
                           path_to_url, is_archive_file,
                           unpack_vcs_link, is_vcs_url, is_file_url,
                           unpack_file_url, unpack_http_url)
+import pip.wheel
+from pip.wheel import move_wheel_files
 
 
 PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt'
@@ -421,6 +423,9 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
 
         pip_egg_info_path = os.path.join(dist.location,
                                          dist.egg_name()) + '.egg-info'
+        dist_info_path = os.path.join(dist.location,
+                                      '-'.join(dist.egg_name().split('-')[:2])
+                                      ) + '.dist-info'
         # workaround for http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=618367
         debian_egg_info_path = pip_egg_info_path.replace(
             '-py%s' % pkg_resources.PY_MAJOR, '')
@@ -429,6 +434,7 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
 
         pip_egg_info_exists = os.path.exists(pip_egg_info_path)
         debian_egg_info_exists = os.path.exists(debian_egg_info_path)
+        dist_info_exists = os.path.exists(dist_info_path)
         if pip_egg_info_exists or debian_egg_info_exists:
             # package installed by pip
             if pip_egg_info_exists:
@@ -472,6 +478,9 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
             easy_install_pth = os.path.join(os.path.dirname(develop_egg_link),
                                             'easy-install.pth')
             paths_to_remove.add_pth(easy_install_pth, dist.location)
+        elif dist_info_exists:
+            for path in pip.wheel.uninstallation_paths(dist):
+                paths_to_remove.add(path)
 
         # find distutils scripts= scripts
         if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'):
@@ -561,6 +570,9 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
         if self.editable:
             self.install_editable(install_options, global_options)
             return
+        if self.is_wheel:
+            self.move_wheel_files(self.source_dir)
+            return
 
         temp_location = tempfile.mkdtemp('-record', 'pip-')
         record_filename = os.path.join(temp_location, 'install-record.txt')
@@ -688,6 +700,10 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
                 self.conflicts_with = existing_dist
         return True
 
+    @property
+    def is_wheel(self):
+        return self.url and '.whl' in self.url
+
     @property
     def is_bundle(self):
         if self._is_bundle is not None:
@@ -756,6 +772,9 @@ exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))
         self._bundle_build_dirs = bundle_build_dirs
         self._bundle_editable_dirs = bundle_editable_dirs
 
+    def move_wheel_files(self, wheeldir):
+        move_wheel_files(self.req, wheeldir)
+
     @property
     def delete_marker_filename(self):
         assert self.source_dir
@@ -803,7 +822,8 @@ class RequirementSet(object):
 
     def __init__(self, build_dir, src_dir, download_dir, download_cache=None,
                  upgrade=False, ignore_installed=False, as_egg=False,
-                 ignore_dependencies=False, force_reinstall=False, use_user_site=False):
+                 ignore_dependencies=False, force_reinstall=False, use_user_site=False,
+                 use_wheel=False):
         self.build_dir = build_dir
         self.src_dir = src_dir
         self.download_dir = download_dir
@@ -821,6 +841,7 @@ class RequirementSet(object):
         self.reqs_to_cleanup = []
         self.as_egg = as_egg
         self.use_user_site = use_user_site
+        self.use_wheel = use_wheel
 
     def __str__(self):
         reqs = [req for req in self.requirements.values()
@@ -977,6 +998,7 @@ class RequirementSet(object):
             logger.indent += 2
             try:
                 is_bundle = False
+                is_wheel = False
                 if req_to_install.editable:
                     if req_to_install.source_dir is None:
                         location = req_to_install.build_location(self.src_dir)
@@ -1027,11 +1049,27 @@ class RequirementSet(object):
                             unpack = False
                     if unpack:
                         is_bundle = req_to_install.is_bundle
+                        is_wheel = url and url.filename.endswith('.whl')
                         if is_bundle:
                             req_to_install.move_bundle_files(self.build_dir, self.src_dir)
                             for subreq in req_to_install.bundle_requirements():
                                 reqs.append(subreq)
                                 self.add_requirement(subreq)
+                        elif is_wheel:
+                            req_to_install.source_dir = location
+                            req_to_install.url = url.url
+                            dist = list(pkg_resources.find_distributions(location))[0]
+                            if not req_to_install.req:
+                                req_to_install.req = dist.as_requirement()
+                                self.add_requirement(req_to_install)
+                            if not self.ignore_dependencies:
+                                for subreq in dist.requires(req_to_install.extras):
+                                    if self.has_requirement(subreq.project_name):
+                                        continue
+                                    subreq = InstallRequirement(str(subreq),
+                                                                req_to_install)
+                                    reqs.append(subreq)
+                                    self.add_requirement(subreq)
                         elif self.is_download:
                             req_to_install.source_dir = location
                             req_to_install.run_egg_info()
@@ -1058,7 +1096,7 @@ class RequirementSet(object):
                                 req_to_install.satisfied_by = None
                             else:
                                 install = False
-                if not is_bundle:
+                if not (is_bundle or is_wheel):
                     ## FIXME: shouldn't be globally added:
                     finder.add_dependency_links(req_to_install.dependency_links)
                     if (req_to_install.extras):

pip/util.py

@@ -442,7 +442,8 @@ def unzip_file(filename, location, flatten=True):
     try:
         zip = zipfile.ZipFile(zipfp)
         leading = has_leading_dir(zip.namelist()) and flatten
-        for name in zip.namelist():
+        for info in zip.infolist():
+            name = info.filename
             data = zip.read(name)
             fn = name
             if leading:
@@ -461,6 +462,7 @@ def unzip_file(filename, location, flatten=True):
                     fp.write(data)
                 finally:
                     fp.close()
+                    os.chmod(fn, info.external_attr >> 16)                
     finally:
         zipfp.close()
 
@@ -546,6 +548,7 @@ def cache_download(target_file, temp_location, content_type):
 
 
 def unpack_file(filename, location, content_type, link):
+    filename = os.path.realpath(filename)
     if (content_type == 'application/zip'
         or filename.endswith('.zip')
         or filename.endswith('.pybundle')

pip/wheel.py

@@ -0,0 +1,183 @@
+"""
+Support functions for installing the "wheel" binary package format.
+"""
+from __future__ import with_statement
+
+import csv
+import os
+import sys
+import shutil
+import functools
+import hashlib
+
+from base64 import urlsafe_b64encode
+
+from pip.util import make_path_relative
+
+def rehash(path, algo='sha256', blocksize=1<<20):
+    """Return (hash, length) for path using hashlib.new(algo)"""
+    h = hashlib.new(algo)
+    length = 0
+    with open(path) as f:
+        block = f.read(blocksize)
+        while block:
+            length += len(block)
+            h.update(block)
+            block = f.read(blocksize)
+    digest = 'sha256='+urlsafe_b64encode(h.digest()).decode('latin1').rstrip('=')
+    return (digest, length)
+
+try:
+    unicode
+    def binary(s):
+        if isinstance(s, unicode):
+            return s.encode('ascii')
+        return s
+except NameError:
+    def binary(s):
+        if isinstance(s, str):
+            return s.encode('ascii')
+
+def open_for_csv(name, mode):
+    if sys.version_info[0] < 3:
+        nl = {}
+        bin = 'b'
+    else:
+        nl = { 'newline': '' }
+        bin = ''
+    return open(name, mode + bin, **nl)
+
+def fix_script(path):
+    """Replace #!python with #!/path/to/python
+    Return True if file was changed."""
+    # XXX RECORD hashes will need to be updated
+    if os.path.isfile(path):
+        script = open(path, 'rb')
+        try:
+            firstline = script.readline()
+            if not firstline.startswith(binary('#!python')):
+                return False
+            exename = sys.executable.encode(sys.getfilesystemencoding())
+            firstline = binary('#!') + sys.executable + binary(os.linesep)
+            rest = script.read()
+        finally:
+            script.close()
+        script = open(path, 'wb')
+        try:
+            script.write(firstline)
+            script.write(rest)
+        finally:
+            script.close()
+        return True
+
+def move_wheel_files(req, wheeldir):
+    from pip.backwardcompat import get_path
+
+    if get_path('purelib') != get_path('platlib'):
+        # XXX check *.dist-info/WHEEL to deal with this obscurity
+        raise NotImplemented("purelib != platlib")
+
+    info_dir = []
+    data_dirs = []
+    source = wheeldir.rstrip(os.path.sep) + os.path.sep
+    location = dest = get_path('platlib')
+    installed = {}
+    changed = set()
+
+    def normpath(src, p):
+        return make_path_relative(src, p).replace(os.path.sep, '/')
+
+    def record_installed(srcfile, destfile, modified=False):
+        """Map archive RECORD paths to installation RECORD paths."""
+        oldpath = normpath(srcfile, wheeldir)
+        newpath = normpath(destfile, location)
+        installed[oldpath] = newpath
+        if modified:
+            changed.add(destfile)
+
+    def clobber(source, dest, is_base, fixer=None):
+        for dir, subdirs, files in os.walk(source):
+            basedir = dir[len(source):].lstrip(os.path.sep)
+            if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
+                continue
+            for s in subdirs:
+                destsubdir = os.path.join(dest, basedir, s)
+                if is_base and basedir == '' and destsubdir.endswith('.data'):
+                    data_dirs.append(s)
+                    continue
+                elif (is_base
+                    and s.endswith('.dist-info')
+                    # is self.req.project_name case preserving?
+                    and s.lower().startswith(req.project_name.replace('-', '_').lower())):
+                    assert not info_dir, 'Multiple .dist-info directories'
+                    info_dir.append(destsubdir)
+                if not os.path.exists(destsubdir):
+                    os.makedirs(destsubdir)
+            for f in files:
+                srcfile = os.path.join(dir, f)
+                destfile = os.path.join(dest, basedir, f)
+                shutil.move(srcfile, destfile)
+                changed = False
+                if fixer:
+                    changed = fixer(destfile)
+                record_installed(srcfile, destfile, changed)
+
+    clobber(source, dest, True)
+
+    assert info_dir, "%s .dist-info directory not found" % req
+
+    for datadir in data_dirs:
+        fixer = None
+        for subdir in os.listdir(os.path.join(wheeldir, datadir)):
+            fixer = None
+            if subdir == 'scripts':
+                fixer = fix_script
+            source = os.path.join(wheeldir, datadir, subdir)
+            dest = get_path(subdir)
+            clobber(source, dest, False, fixer=fixer)
+
+    record = os.path.join(info_dir[0], 'RECORD')
+    temp_record = os.path.join(info_dir[0], 'RECORD.pip')
+    with open_for_csv(record, 'r') as record_in:
+        with open_for_csv(temp_record, 'w+') as record_out:
+            reader = csv.reader(record_in)
+            writer = csv.writer(record_out)
+            for row in reader:
+                row[0] = installed.pop(row[0], row[0])
+                if row[0] in changed:
+                    row[1], row[2] = rehash(row[0])
+                writer.writerow(row)
+            for f in installed:
+                writer.writerow((installed[f], '', ''))
+    shutil.move(temp_record, record)
+
+def _unique(fn):
+    @functools.wraps(fn)
+    def unique(*args, **kw):
+        seen = set()
+        for item in fn(*args, **kw):
+            if item not in seen:
+                seen.add(item)
+                yield item
+    return unique
+
+@_unique
+def uninstallation_paths(dist):
+    """
+    Yield all the uninstallation paths for dist based on RECORD-without-.pyc
+
+    Yield paths to all the files in RECORD. For each .py file in RECORD, add
+    the .pyc in the same directory.
+
+    UninstallPathSet.add() takes care of the __pycache__ .pyc.
+    """
+    from pip.req import FakeFile # circular import
+    r = csv.reader(FakeFile(dist.get_metadata_lines('RECORD')))
+    for row in r:
+        path = os.path.join(dist.location, row[0])
+        yield path
+        if path.endswith('.py'):
+            dn, fn = os.path.split(path)
+            base = fn[:-3]
+            path = os.path.join(dn, base+'.pyc')
+            yield path

tests/test_basic.py

@@ -326,6 +326,63 @@ def test_install_as_egg():
     assert join(egg_folder, 'fspkg') in result.files_created, str(result)
 
 
+def test_install_from_wheel():
+    """
+    Test installing from a wheel.
+    """
+    env = reset_env(use_distribute=True)
+    result = run_pip('install', 'markerlib', expect_error=False)
+    find_links = 'file://'+abspath(join(here, 'packages'))
+    result = run_pip('install', 'simple.dist', '--use-wheel',
+                     '--no-index', '--find-links='+find_links,
+                     expect_error=False)
+    dist_info_folder = env.site_packages/'simple.dist-0.1.dist-info'
+    assert dist_info_folder in result.files_created, (dist_info_folder,
+                                                      result.files_created,
+                                                      result.stdout)
+
+
+def test_install_from_wheel_with_extras():
+    """
+    Test installing from a wheel.
+    """
+    from nose import SkipTest
+    try:
+        import ast
+    except ImportError:
+        raise SkipTest("Need ast module to interpret wheel extras")
+    env = reset_env(use_distribute=True)
+    result = run_pip('install', 'markerlib', expect_error=False)
+    find_links = 'file://'+abspath(join(here, 'packages'))
+    result = run_pip('install', 'complex-dist[simple]', '--use-wheel',
+                     '--no-index', '--find-links='+find_links,
+                     expect_error=False)
+    dist_info_folder = env.site_packages/'complex_dist-0.1.dist-info'
+    assert dist_info_folder in result.files_created, (dist_info_folder,
+                                                      result.files_created,
+                                                      result.stdout)
+    dist_info_folder = env.site_packages/'simple.dist-0.1.dist-info'
+    assert dist_info_folder in result.files_created, (dist_info_folder,
+                                                      result.files_created,
+                                                      result.stdout)
+
+
+def test_install_from_wheel_file():
+    """
+    Test installing directly from a wheel file.
+    """
+    env = reset_env(use_distribute=True)
+    result = run_pip('install', 'markerlib', expect_error=False)
+    package = abspath(join(here,
+                           'packages',
+                           'simple.dist-0.1-py2.py3-none-any.whl'))
+    result = run_pip('install', package, '--no-index', expect_error=False)
+    dist_info_folder = env.site_packages/'simple.dist-0.1.dist-info'
+    assert dist_info_folder in result.files_created, (dist_info_folder,
+                                                      result.files_created,
+                                                      result.stdout)
+
+
 def test_install_curdir():
     """
     Test installing current directory ('.').

tests/test_wheel.py

@@ -0,0 +1,30 @@
+"""Tests for wheel binary packages and .dist-info."""
+
+import imp
+from pip import wheel
+
+def test_uninstallation_paths():
+    class dist(object):
+        def get_metadata_lines(self, record):
+            return ['file.py,,',
+                    'file.pyc,,',
+                    'file.so,,',
+                    'nopyc.py']
+        location = ''
+
+    d = dist()
+
+    paths = list(wheel.uninstallation_paths(d))
+
+    expected = ['file.py',
+                'file.pyc',
+                'file.so',
+                'nopyc.py',
+                'nopyc.pyc']
+
+    assert paths == expected
+
+    # Avoid an easy 'unique generator' bug
+    paths2 = list(wheel.uninstallation_paths(d))
+
+    assert paths2 == paths