From 9a4be9b2815e69b130329963d4fc846850364f64 Mon Sep 17 00:00:00 2001 From: Mathew Jennings Date: Mon, 15 Jun 2015 17:36:59 -0400 Subject: [PATCH 1/8] Add --platform and --python-version to `pip download`. With the --platform option, a user can download wheels with a different platform than that of the local machine running the command. With the --python-version option, a user can download wheels that are explicitly compatible with a specific Python interpreter version. This functionality is meant for utilities that gather dependencies and prepare distributions for other platforms. --- pip/basecommand.py | 5 +++- pip/commands/download.py | 42 +++++++++++++++++++++++++++++-- pip/index.py | 41 ++++++++++++++++++++++++------ pip/pep425tags.py | 6 +++-- tests/functional/test_download.py | 26 +++++++++++++++++++ tests/unit/test_finder.py | 13 +++++----- 6 files changed, 113 insertions(+), 20 deletions(-) diff --git a/pip/basecommand.py b/pip/basecommand.py index 261d3999a..a3d21c7bb 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -311,7 +311,8 @@ class RequirementCommand(Command): 'to %(name)s (see "pip help %(name)s")' % opts) logger.warning(msg) - def _build_package_finder(self, options, session): + def _build_package_finder(self, options, session, + platform=None, desired_interp_versions=None): """ Create a package finder appropriate to this requirement command. """ @@ -328,4 +329,6 @@ class RequirementCommand(Command): allow_all_prereleases=options.pre, process_dependency_links=options.process_dependency_links, session=session, + platform=platform, + versions=desired_interp_versions, ) diff --git a/pip/commands/download.py b/pip/commands/download.py index 4155e052c..a33e89395 100644 --- a/pip/commands/download.py +++ b/pip/commands/download.py @@ -6,6 +6,7 @@ import os from pip.req import RequirementSet from pip.basecommand import RequirementCommand from pip import cmdoptions +from pip.pep425tags import get_platform from pip.utils import ensure_dir, normalize_path from pip.utils.build import BuildDirectory from pip.utils.filesystem import check_path_owner @@ -63,6 +64,33 @@ class DownloadCommand(RequirementCommand): help=("Download packages into ."), ) + cmd_opts.add_option( + '--platform', + dest='platform', + metavar='platform', + default=get_platform(), + help=("Specifically download wheels compatible with " + "where the default is the platfrom of the local computer. " + "This option may only be used if --download is also being " + "used."), + ) + + cmd_opts.add_option( + '--interpreter-version', + dest='interpreter_version', + metavar='version', + default='', + help=("Specifically download wheels compatible with Python " + "interpreter version . If not specified, then the " + "current system interpreter version is used. This option " + "may only be used if --download is also being used. This " + "is a stricter approach compared to the native search " + "performed without this option specified: this does not " + "accept previous minor versions of Python. It is meant to " + "be used when you want to download packages that support an " + "exact version of Python."), + ) + index_opts = cmdoptions.make_option_group( cmdoptions.non_deprecated_index_group, self.parser, @@ -73,14 +101,24 @@ class DownloadCommand(RequirementCommand): def run(self, options, args): options.ignore_installed = True + + if options.interpreter_version: + desired_interp_versions = [options.interpreter_version] + else: + desired_interp_versions = None + options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) ensure_dir(options.download_dir) with self._build_session(options) as session: - - finder = self._build_package_finder(options, session) + finder = self._build_package_finder( + options=options, + session=session, + platform=options.platform, + desired_interp_versions=desired_interp_versions, + ) build_delete = (not (options.no_clean or options.build_dir)) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( diff --git a/pip/index.py b/pip/index.py index f8a298e9f..08e23250d 100644 --- a/pip/index.py +++ b/pip/index.py @@ -28,7 +28,7 @@ from pip.exceptions import ( ) from pip.download import HAS_TLS, is_url, path_to_url, url_to_path from pip.wheel import Wheel, wheel_ext -from pip.pep425tags import supported_tags +from pip.pep425tags import get_platform, get_supported from pip._vendor import html5lib, requests, six from pip._vendor.packaging.version import parse as parse_version from pip._vendor.packaging.utils import canonicalize_name @@ -104,12 +104,20 @@ class PackageFinder(object): def __init__(self, find_links, index_urls, allow_all_prereleases=False, trusted_hosts=None, process_dependency_links=False, - session=None, format_control=None): + session=None, format_control=None, platform=None, + versions=None): """Create a PackageFinder. :param format_control: A FormatControl object or None. Used to control the selection of source packages / binary packages when consulting the index and links. + :param platform: A string or None. If None, searches for packages + that are supported by the current system. Otherwise, will find + packages that can be built on the platform passed in. It is + understood that these packages will only be downloaded for + distribution: they will not be built locally. + :param versions: A list of strings or None. This is passed directly + to pep425tags.py in the get_supported() method. """ if session is None: raise TypeError( @@ -153,6 +161,19 @@ class PackageFinder(object): # The Session we'll use to make requests self.session = session + # The platform for which to find compatible packages + self.platform = platform or get_platform() + + # The valid tags to check potential found wheel candidates against + self.valid_tags = get_supported( + versions=versions, + specificplatform=self.platform + ) + self.valid_tags_noarch = get_supported( + versions=versions, + noarch=True + ) + # If we don't have TLS enabled, then WARN if anyplace we're looking # relies on TLS. if not HAS_TLS: @@ -236,22 +257,24 @@ class PackageFinder(object): If not finding wheels, then sorted by version only. If finding wheels, then the sort order is by version, then: 1. existing installs - 2. wheels ordered via Wheel.support_index_min() + 2. wheels ordered via Wheel.support_index_min(self.valid_tags) 3. source archives Note: it was considered to embed this logic into the Link comparison operators, but then different sdist links with the same version, would have to be considered equal """ - support_num = len(supported_tags) - if candidate.location.is_wheel: + support_num = len(self.valid_tags) + if candidate.location == INSTALLED_VERSION: + pri = 1 + elif candidate.location.is_wheel: # can raise InvalidWheelFilename wheel = Wheel(candidate.location.filename) - if not wheel.supported(): + if not wheel.supported(self.valid_tags): raise UnsupportedWheel( "%s is not a supported wheel for this platform. It " "can't be sorted." % wheel.filename ) - pri = -(wheel.support_index_min()) + pri = -(wheel.support_index_min(self.valid_tags)) else: # sdist pri = -(support_num) return (candidate.version, pri) @@ -581,6 +604,7 @@ class PackageFinder(object): def _link_package_versions(self, link, search): """Return an InstallationCandidate or None""" + platform = self.platform version = None if link.egg_fragment: @@ -612,7 +636,8 @@ class PackageFinder(object): self._log_skipped_link( link, 'wrong project name (not %s)' % search.supplied) return - if not wheel.supported(): + + if not wheel.supported(self.valid_tags): self._log_skipped_link( link, 'it is not compatible with this Python') return diff --git a/pip/pep425tags.py b/pip/pep425tags.py index dd0ca7399..951212ca3 100644 --- a/pip/pep425tags.py +++ b/pip/pep425tags.py @@ -223,12 +223,14 @@ def get_darwin_arches(major, minor, machine): return arches -def get_supported(versions=None, noarch=False): +def get_supported(versions=None, noarch=False, specificplatform=None): """Return a list of supported tags for each version specified in `versions`. :param versions: a list of string versions, of the form ["33", "32"], or None. The first version will be assumed to support our ABI. + :param specificplatform: specify the exact platform you want valid + tags for, or None. If None, use the local system platform. """ supported = [] @@ -260,7 +262,7 @@ def get_supported(versions=None, noarch=False): abis.append('none') if not noarch: - arch = get_platform() + arch = specificplatform or get_platform() if sys.platform == 'darwin': # support macosx-10.6-intel on macosx-10.9-x86_64 match = _osx_arch_pat.match(arch) diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 54132261f..f2d0bd057 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -159,3 +159,29 @@ def test_download_vcs_link(script): in result.files_created ) assert script.site_packages / 'piptestpackage' not in result.files_created + + +@pytest.mark.network +def test_download_specify_platform(script, data): + """ + Test using "pip download --platform" to download a .whl archive + supported for a specific platform + """ + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--dest', '.', '--platform', 'linux_x86_64', 'simplewheel' + ) + assert ( + Path('scratch') / 'simplewheel-2.0-py2.py3-none-any.whl' + in result.files_created + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--dest', '.', '--platform', 'macosx_10_9_x86_64', + 'requires_source' + ) + assert ( + Path('scratch') / 'requires_source-1.0-py2.py3-none-any.whl' + in result.files_created + ) diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 48265e98d..b8d44f2e3 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -149,6 +149,7 @@ class TestWheel: [], session=PipSession(), ) + finder.valid_tags = pip.pep425tags.supported_tags with pytest.raises(DistributionNotFound): finder.find_requirement(req, True) @@ -210,11 +211,6 @@ class TestWheel: with pytest.raises(BestVersionAlreadyInstalled): finder.find_requirement(req, True) - @patch('pip.pep425tags.supported_tags', [ - ('pyT', 'none', 'TEST'), - ('pyT', 'TEST', 'any'), - ('pyT', 'none', 'any'), - ]) def test_link_sorting(self): """ Test link sorting @@ -242,9 +238,12 @@ class TestWheel: Link('simple-1.0.tar.gz'), ), ] - finder = PackageFinder([], [], session=PipSession()) - + finder.valid_tags = [ + ('pyT', 'none', 'TEST'), + ('pyT', 'TEST', 'any'), + ('pyT', 'none', 'any'), + ] results = sorted(links, key=finder._candidate_sort_key, reverse=True) results2 = sorted(reversed(links), From bbe99ce62d626f6c01552c3fcab953b014674c53 Mon Sep 17 00:00:00 2001 From: Patrick Lawson Date: Sun, 20 Dec 2015 20:20:59 -0500 Subject: [PATCH 2/8] Add --implementation and --abi parameters to ``pip download``. Add thorough tests, usage documentation, and handle some merge issues. --- CHANGES.txt | 5 + docs/reference/pip_download.rst | 83 ++++++- pip/basecommand.py | 7 +- pip/commands/download.py | 82 +++++-- pip/index.py | 23 +- pip/pep425tags.py | 19 +- tests/functional/test_download.py | 389 +++++++++++++++++++++++++++++- 7 files changed, 550 insertions(+), 58 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a5189c4a3..f28c39d56 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,6 +12,11 @@ editable dependencies before); this allows running pip wheel on the result of pip freeze in presence of editable requirements (:issue:`3291`) +* Add --platform, --python-version, --implementation, --abi, + and --manylinux1 parameters to ``pip download``. + These allow utilities and advanced users to gather distributions for + interpreters other than the one pip is being run on. + **8.1.2 (2016-05-10)** diff --git a/docs/reference/pip_download.rst b/docs/reference/pip_download.rst index b1f805fea..a0b7267f0 100644 --- a/docs/reference/pip_download.rst +++ b/docs/reference/pip_download.rst @@ -26,9 +26,22 @@ which is now deprecated and will be removed in pip 10. ``pip download`` does the same resolution and downloading as ``pip install``, but instead of installing the dependencies, it collects the downloaded distributions into the directory provided (defaulting to the current -directory). This directory can later be passed as the value to -``pip install --find-links`` to facilitate offline or locked down package -installation. +directory). This directory can later be passed as the value to ``pip install +--find-links`` to facilitate offline or locked down package installation. + +``pip download`` with the ``--platform``, ``--python-version``, +``--implementation``, and ``--abi`` options provides the ability to fetch +dependencies for an interpreter and system other than the ones that pip is +running on. ``--only-binary=:all:`` is required when using any of these +options. It is important to note that these options all default to the +current system/interpreter, and not to the most restrictive constraints (e.g. +platform any, abi none, etc). To avoid fetching dependencies that happen to +match the constraint of the current interpreter (but not your target one), it +is recommended to specify all of these options if you are specifying one of +them. Generic dependencies (e.g. universal wheels, or dependencies with no +platform, abi, or implementation constraints) will still match an over- +constrained download requirement. + Options @@ -44,8 +57,64 @@ Examples #. Download a package and all of its dependencies - :: + :: - $ pip download SomePackage - $ pip download -d . SomePackage # equivalent to above - $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + $ pip download SomePackage + $ pip download -d . SomePackage # equivalent to above + $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + +#. Download a package and all of its dependencies with OSX specific interpreter constraints. + This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, + this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, + etc. + It will also match deps with platform ``any``. Also force the interpreter version to ``27`` + (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). + + :: + + $ pip download \ + --only-binary=:all: \ + --platform macosx-10_10_x86_64 \ + --python-version 27 \ + --implementation cp \ + SomePackage + +#. Download a package and its dependencies with linux specific constraints, including + packages that support the ``manylinux1`` platform. Force the interpreter to be any + minor version of py3k, and only accept ``cp34m`` or ``none`` as the abi. + + :: + + $ pip download \ + --only-binary=:all: \ + --platform linux_x86_64 --manylinux \ + --python-version 3 \ + --implementation cp \ + --abi cp34m \ + SomePackage + +#. Force platform, implementation, and abi agnostic deps. + + :: + + $ pip download \ + --only-binary=:all: \ + --platform any \ + --python-version 3 \ + --implementation py \ + --abi none \ + SomePackage + +#. Even when overconstrained, this will still correctly fetch the pip universal wheel. + + :: + + $ pip download \ + --only-binary=:all: \ + --platform linux_x86_64 --manylinux \ + --python-version 33 \ + --implementation cp \ + --abi cp34m \ + pip>=8 + $ ls pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl diff --git a/pip/basecommand.py b/pip/basecommand.py index a3d21c7bb..54c670672 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -312,7 +312,8 @@ class RequirementCommand(Command): logger.warning(msg) def _build_package_finder(self, options, session, - platform=None, desired_interp_versions=None): + platform=None, python_versions=None, + abi=None, implementation=None): """ Create a package finder appropriate to this requirement command. """ @@ -330,5 +331,7 @@ class RequirementCommand(Command): process_dependency_links=options.process_dependency_links, session=session, platform=platform, - versions=desired_interp_versions, + versions=python_versions, + abi=abi, + implementation=implementation, ) diff --git a/pip/commands/download.py b/pip/commands/download.py index a33e89395..8e7fbca39 100644 --- a/pip/commands/download.py +++ b/pip/commands/download.py @@ -3,10 +3,11 @@ from __future__ import absolute_import import logging import os +from pip.exceptions import CommandError +from pip.index import FormatControl from pip.req import RequirementSet from pip.basecommand import RequirementCommand from pip import cmdoptions -from pip.pep425tags import get_platform from pip.utils import ensure_dir, normalize_path from pip.utils.build import BuildDirectory from pip.utils.filesystem import check_path_owner @@ -68,27 +69,47 @@ class DownloadCommand(RequirementCommand): '--platform', dest='platform', metavar='platform', - default=get_platform(), - help=("Specifically download wheels compatible with " - "where the default is the platfrom of the local computer. " - "This option may only be used if --download is also being " - "used."), + default=None, + help=("Only download wheels compatible with . " + "Defaults to the platform of the local computer."), ) cmd_opts.add_option( - '--interpreter-version', - dest='interpreter_version', - metavar='version', - default='', - help=("Specifically download wheels compatible with Python " + '--python-version', + dest='python_version', + metavar='python_version', + default=None, + help=("Only download wheels compatible with Python " "interpreter version . If not specified, then the " - "current system interpreter version is used. This option " - "may only be used if --download is also being used. This " - "is a stricter approach compared to the native search " - "performed without this option specified: this does not " - "accept previous minor versions of Python. It is meant to " - "be used when you want to download packages that support an " - "exact version of Python."), + "current system interpreter minor version is used. A major " + "version (e.g. '2') can be specified to match all " + "minor revs of that major version. A minor version " + "(e.g. '34') can also be specified."), + ) + + cmd_opts.add_option( + '--implementation', + dest='implementation', + metavar='implementation', + default=None, + help=("Only download wheels compatible with Python " + "implementation , e.g. 'pp', 'jy', 'cp', " + " or 'ip'. If not specified, then the current " + "interpreter implementation is used. Use 'py' to force " + "implementation-agnostic wheels."), + ) + + cmd_opts.add_option( + '--abi', + dest='abi', + metavar='abi', + default=None, + help=("Only download wheels compatible with Python " + "abi , e.g. 'pypy_41'. If not specified, then the " + "current interpreter abi tag is used. Generally " + "you will need to specify --implementation, " + "--platform, and --python-version when using " + "this option."), ) index_opts = cmdoptions.make_option_group( @@ -102,10 +123,25 @@ class DownloadCommand(RequirementCommand): def run(self, options, args): options.ignore_installed = True - if options.interpreter_version: - desired_interp_versions = [options.interpreter_version] + if options.python_version: + python_versions = [options.python_version] else: - desired_interp_versions = None + python_versions = None + + dist_restriction_set = any([ + options.python_version, + options.platform, + options.abi, + options.implementation, + ]) + binary_only = FormatControl(set(), set([':all:'])) + if dist_restriction_set and options.format_control != binary_only: + raise CommandError( + "--only-binary=:all: must be set and --no-binary must not " + "be set (or must be set to :none:) when restricting platform " + "and interpreter constraints using --python-version, " + "--platform, --abi, or --implementation." + ) options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) @@ -117,7 +153,9 @@ class DownloadCommand(RequirementCommand): options=options, session=session, platform=options.platform, - desired_interp_versions=desired_interp_versions, + python_versions=python_versions, + abi=options.abi, + implementation=options.implementation, ) build_delete = (not (options.no_clean or options.build_dir)) if options.cache_dir and not check_path_owner(options.cache_dir): diff --git a/pip/index.py b/pip/index.py index 08e23250d..b7589d66b 100644 --- a/pip/index.py +++ b/pip/index.py @@ -28,7 +28,7 @@ from pip.exceptions import ( ) from pip.download import HAS_TLS, is_url, path_to_url, url_to_path from pip.wheel import Wheel, wheel_ext -from pip.pep425tags import get_platform, get_supported +from pip.pep425tags import get_supported from pip._vendor import html5lib, requests, six from pip._vendor.packaging.version import parse as parse_version from pip._vendor.packaging.utils import canonicalize_name @@ -105,7 +105,7 @@ class PackageFinder(object): def __init__(self, find_links, index_urls, allow_all_prereleases=False, trusted_hosts=None, process_dependency_links=False, session=None, format_control=None, platform=None, - versions=None): + versions=None, abi=None, implementation=None): """Create a PackageFinder. :param format_control: A FormatControl object or None. Used to control @@ -118,6 +118,12 @@ class PackageFinder(object): distribution: they will not be built locally. :param versions: A list of strings or None. This is passed directly to pep425tags.py in the get_supported() method. + :param abi: A string or None. This is passed directly + to pep425tags.py in the get_supported() method. + :param implementation: A string or None. This is passed directly + to pep425tags.py in the get_supported() method. + :param manylinux1: A boolean or None. This is passed directly + to pep425tags.py in the get_supported() method. """ if session is None: raise TypeError( @@ -161,13 +167,12 @@ class PackageFinder(object): # The Session we'll use to make requests self.session = session - # The platform for which to find compatible packages - self.platform = platform or get_platform() - # The valid tags to check potential found wheel candidates against self.valid_tags = get_supported( versions=versions, - specificplatform=self.platform + platform=platform, + abi=abi, + impl=implementation, ) self.valid_tags_noarch = get_supported( versions=versions, @@ -264,9 +269,7 @@ class PackageFinder(object): with the same version, would have to be considered equal """ support_num = len(self.valid_tags) - if candidate.location == INSTALLED_VERSION: - pri = 1 - elif candidate.location.is_wheel: + if candidate.location.is_wheel: # can raise InvalidWheelFilename wheel = Wheel(candidate.location.filename) if not wheel.supported(self.valid_tags): @@ -604,8 +607,6 @@ class PackageFinder(object): def _link_package_versions(self, link, search): """Return an InstallationCandidate or None""" - platform = self.platform - version = None if link.egg_fragment: egg_info = link.egg_fragment diff --git a/pip/pep425tags.py b/pip/pep425tags.py index 951212ca3..96e9d1691 100644 --- a/pip/pep425tags.py +++ b/pip/pep425tags.py @@ -223,14 +223,19 @@ def get_darwin_arches(major, minor, machine): return arches -def get_supported(versions=None, noarch=False, specificplatform=None): +def get_supported(versions=None, noarch=False, platform=None, + impl=None, abi=None): """Return a list of supported tags for each version specified in `versions`. :param versions: a list of string versions, of the form ["33", "32"], or None. The first version will be assumed to support our ABI. - :param specificplatform: specify the exact platform you want valid + :param platform: specify the exact platform you want valid tags for, or None. If None, use the local system platform. + :param impl: specify the exact implementation you want valid + tags for, or None. If None, use the local interpreter impl. + :param abi: specify the exact abi you want valid + tags for, or None. If None, use the local interpreter abi. """ supported = [] @@ -243,11 +248,11 @@ def get_supported(versions=None, noarch=False, specificplatform=None): for minor in range(version_info[-1], -1, -1): versions.append(''.join(map(str, major + (minor,)))) - impl = get_abbr_impl() + impl = impl or get_abbr_impl() abis = [] - abi = get_abi_tag() + abi = abi or get_abi_tag() if abi: abis[0:0] = [abi] @@ -262,8 +267,8 @@ def get_supported(versions=None, noarch=False, specificplatform=None): abis.append('none') if not noarch: - arch = specificplatform or get_platform() - if sys.platform == 'darwin': + arch = platform or get_platform() + if arch.startswith('macosx'): # support macosx-10.6-intel on macosx-10.9-x86_64 match = _osx_arch_pat.match(arch) if match: @@ -276,7 +281,7 @@ def get_supported(versions=None, noarch=False, specificplatform=None): else: # arch pattern didn't match (?!) arches = [arch] - elif is_manylinux1_compatible(): + elif platform is None and is_manylinux1_compatible(): arches = [arch.replace('linux', 'manylinux1'), arch] else: arches = [arch] diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index f2d0bd057..16da5f2d4 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -5,6 +5,12 @@ import pytest from tests.lib.path import Path +def fake_wheel(data, wheel_path): + data.packages.join( + 'simple.dist-0.1-py2.py3-none-any.whl' + ).copy(data.packages.join(wheel_path)) + + @pytest.mark.network def test_download_if_requested(script): """ @@ -161,27 +167,392 @@ def test_download_vcs_link(script): assert script.site_packages / 'piptestpackage' not in result.files_created -@pytest.mark.network -def test_download_specify_platform(script, data): +def test_download_specify_platform_only_binary(script, data): """ - Test using "pip download --platform" to download a .whl archive - supported for a specific platform + Confirm that specifying an interpreter/platform constraint + enforces that ``--only-binary=:all:`` is set. """ + fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + result = script.pip( 'download', '--no-index', '--find-links', data.find_links, - '--dest', '.', '--platform', 'linux_x86_64', 'simplewheel' + '--only-binary=:all:', + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake' ) assert ( - Path('scratch') / 'simplewheel-2.0-py2.py3-none-any.whl' + Path('scratch') / 'fake-1.0-py2.py3-none-any.whl' in result.files_created ) result = script.pip( 'download', '--no-index', '--find-links', data.find_links, - '--dest', '.', '--platform', 'macosx_10_9_x86_64', - 'requires_source' + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake', + expect_error=True, + ) + assert '--only-binary=:all:' in result.stderr + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake', + expect_error=True, + ) + assert '--only-binary=:all:' in result.stderr + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--no-binary=fake', + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake', + expect_error=True, + ) + assert '--only-binary=:all:' in result.stderr + + +def test_download_specify_platform(script, data): + """ + Test using "pip download --platform" to download a .whl archive + supported for a specific platform + """ + fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + + # Confirm that universal wheels are returned even for specific + # platforms. + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake' ) assert ( - Path('scratch') / 'requires_source-1.0-py2.py3-none-any.whl' + Path('scratch') / 'fake-1.0-py2.py3-none-any.whl' in result.files_created ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'macosx_10_9_x86_64', + 'fake' + ) + + data.reset() + fake_wheel(data, 'fake-1.0-py2.py3-none-macosx_10_9_x86_64.whl') + fake_wheel(data, 'fake-2.0-py2.py3-none-linux_x86_64.whl') + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'macosx_10_10_x86_64', + 'fake' + ) + assert ( + Path('scratch') / + 'fake-1.0-py2.py3-none-macosx_10_9_x86_64.whl' + in result.files_created + ) + + # OSX platform wheels are not backward-compatible. + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'macosx_10_8_x86_64', + 'fake', + expect_error=True, + ) + + # No linux wheel provided for this version. + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake==1', + expect_error=True, + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake==2' + ) + assert ( + Path('scratch') / 'fake-2.0-py2.py3-none-linux_x86_64.whl' + in result.files_created + ) + + +def test_download_platform_manylinux(script, data): + """ + Test using "pip download --platform" to download a .whl archive + supported for a specific platform. + """ + fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + # Confirm that universal wheels are returned even for specific + # platforms. + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake', + ) + assert ( + Path('scratch') / 'fake-1.0-py2.py3-none-any.whl' + in result.files_created + ) + + data.reset() + fake_wheel(data, 'fake-1.0-py2.py3-none-manylinux1_x86_64.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'manylinux1_x86_64', + 'fake', + ) + assert ( + Path('scratch') / + 'fake-1.0-py2.py3-none-manylinux1_x86_64.whl' + in result.files_created + ) + + # When specifying the platform, manylinux1 needs to be the + # explicit platform--it won't ever be added to the compatible + # tags. + data.reset() + fake_wheel(data, 'fake-1.0-py2.py3-none-linux_x86_64.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'linux_x86_64', + 'fake', + expect_error=True, + ) + + +def test_download_specify_python_version(script, data): + """ + Test using "pip download --python-version" to download a .whl archive + supported for a specific interpreter + """ + fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '2', + 'fake' + ) + assert ( + Path('scratch') / 'fake-1.0-py2.py3-none-any.whl' + in result.files_created + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '3', + 'fake' + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '27', + 'fake' + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '33', + 'fake' + ) + + data.reset() + fake_wheel(data, 'fake-1.0-py2-none-any.whl') + fake_wheel(data, 'fake-2.0-py3-none-any.whl') + + # No py3 provided for version 1. + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '3', + 'fake==1.0', + expect_error=True, + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '2', + 'fake' + ) + assert ( + Path('scratch') / 'fake-1.0-py2-none-any.whl' + in result.files_created + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '26', + 'fake' + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '3', + 'fake' + ) + assert ( + Path('scratch') / 'fake-2.0-py3-none-any.whl' + in result.files_created + ) + + +def test_download_specify_abi(script, data): + """ + Test using "pip download --abi" to download a .whl archive + supported for a specific abi + """ + fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--implementation', 'fk', + '--abi', 'fake_abi', + 'fake' + ) + assert ( + Path('scratch') / 'fake-1.0-py2.py3-none-any.whl' + in result.files_created + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--implementation', 'fk', + '--abi', 'none', + 'fake' + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--abi', 'cp27m', + 'fake', + expect_error=True, + ) + + data.reset() + fake_wheel(data, 'fake-1.0-fk2-fakeabi-fake_platform.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '2', + '--implementation', 'fk', + '--platform', 'fake_platform', + '--abi', 'fakeabi', + 'fake' + ) + assert ( + Path('scratch') / 'fake-1.0-fk2-fakeabi-fake_platform.whl' + in result.files_created + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--implementation', 'fk', + '--platform', 'fake_platform', + '--abi', 'none', + 'fake', + expect_error=True, + ) + + +def test_download_specify_implementation(script, data): + """ + Test using "pip download --abi" to download a .whl archive + supported for a specific abi + """ + fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--implementation', 'fk', + 'fake' + ) + assert ( + Path('scratch') / 'fake-1.0-py2.py3-none-any.whl' + in result.files_created + ) + + data.reset() + fake_wheel(data, 'fake-1.0-fk2.fk3-none-any.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--implementation', 'fk', + 'fake' + ) + assert ( + Path('scratch') / 'fake-1.0-fk2.fk3-none-any.whl' + in result.files_created + ) + + data.reset() + fake_wheel(data, 'fake-1.0-fk3-none-any.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--implementation', 'fk', + '--python-version', '3', + 'fake' + ) + assert ( + Path('scratch') / 'fake-1.0-fk3-none-any.whl' + in result.files_created + ) + + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--implementation', 'fk', + '--python-version', '2', + 'fake', + expect_error=True, + ) From 5c8e31fc11d71f3d0b243c4f8082d9ab756543b7 Mon Sep 17 00:00:00 2001 From: Patrick Lawson Date: Wed, 11 May 2016 12:33:06 -0400 Subject: [PATCH 3/8] Fix CHANGES entry --- CHANGES.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f28c39d56..67163d0b3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,10 +12,10 @@ editable dependencies before); this allows running pip wheel on the result of pip freeze in presence of editable requirements (:issue:`3291`) -* Add --platform, --python-version, --implementation, --abi, - and --manylinux1 parameters to ``pip download``. - These allow utilities and advanced users to gather distributions for - interpreters other than the one pip is being run on. +* Add --platform, --python-version, --implementation and --abi parameters to + ``pip download``. These allow utilities and advanced users to gather + distributions for interpreters other than the one pip is being run on. + (:pull:`3092`) **8.1.2 (2016-05-10)** From d76477d0b847ecc2258d2fbcd7942812fcf32925 Mon Sep 17 00:00:00 2001 From: Patrick Lawson Date: Fri, 27 May 2016 17:00:43 -0400 Subject: [PATCH 4/8] Fix docs example enumeration --- docs/reference/pip_download.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/pip_download.rst b/docs/reference/pip_download.rst index a0b7267f0..bf3e4c989 100644 --- a/docs/reference/pip_download.rst +++ b/docs/reference/pip_download.rst @@ -55,7 +55,7 @@ Options Examples ******** -#. Download a package and all of its dependencies +1. Download a package and all of its dependencies :: @@ -63,7 +63,7 @@ Examples $ pip download -d . SomePackage # equivalent to above $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage -#. Download a package and all of its dependencies with OSX specific interpreter constraints. +2. Download a package and all of its dependencies with OSX specific interpreter constraints. This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, etc. @@ -79,7 +79,7 @@ Examples --implementation cp \ SomePackage -#. Download a package and its dependencies with linux specific constraints, including +3. Download a package and its dependencies with linux specific constraints, including packages that support the ``manylinux1`` platform. Force the interpreter to be any minor version of py3k, and only accept ``cp34m`` or ``none`` as the abi. @@ -93,7 +93,7 @@ Examples --abi cp34m \ SomePackage -#. Force platform, implementation, and abi agnostic deps. +4. Force platform, implementation, and abi agnostic deps. :: @@ -105,7 +105,7 @@ Examples --abi none \ SomePackage -#. Even when overconstrained, this will still correctly fetch the pip universal wheel. +5. Even when overconstrained, this will still correctly fetch the pip universal wheel. :: From d61148c55d1eec34e3f9d4a980e78246b76ce41a Mon Sep 17 00:00:00 2001 From: Patrick Lawson Date: Fri, 27 May 2016 17:02:33 -0400 Subject: [PATCH 5/8] Fix PR reference --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 67163d0b3..0537acc7f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -15,7 +15,7 @@ * Add --platform, --python-version, --implementation and --abi parameters to ``pip download``. These allow utilities and advanced users to gather distributions for interpreters other than the one pip is being run on. - (:pull:`3092`) + (:pull:`3760`) **8.1.2 (2016-05-10)** From 17ffa30edc7c4026467311368078b54b3dbcffd1 Mon Sep 17 00:00:00 2001 From: Patrick Lawson Date: Thu, 2 Jun 2016 10:55:29 -0400 Subject: [PATCH 6/8] Fix old docstring manylinux1 reference. --- pip/index.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pip/index.py b/pip/index.py index b7589d66b..db08b0016 100644 --- a/pip/index.py +++ b/pip/index.py @@ -113,17 +113,15 @@ class PackageFinder(object): the index and links. :param platform: A string or None. If None, searches for packages that are supported by the current system. Otherwise, will find - packages that can be built on the platform passed in. It is - understood that these packages will only be downloaded for - distribution: they will not be built locally. + packages that can be built on the platform passed in. These + packages will only be downloaded for distribution: they will + not be built locally. :param versions: A list of strings or None. This is passed directly to pep425tags.py in the get_supported() method. :param abi: A string or None. This is passed directly to pep425tags.py in the get_supported() method. :param implementation: A string or None. This is passed directly to pep425tags.py in the get_supported() method. - :param manylinux1: A boolean or None. This is passed directly - to pep425tags.py in the get_supported() method. """ if session is None: raise TypeError( From 5337d0150261806383be84102d6aab9d31a2951f Mon Sep 17 00:00:00 2001 From: Patrick Lawson Date: Sun, 5 Jun 2016 16:49:03 -0400 Subject: [PATCH 7/8] Remove dead code, remove manylinux reference, fix docs enumeration. --- docs/reference/pip_download.rst | 98 ++++++++++++++++----------------- pip/index.py | 4 -- 2 files changed, 49 insertions(+), 53 deletions(-) diff --git a/docs/reference/pip_download.rst b/docs/reference/pip_download.rst index bf3e4c989..466b3e577 100644 --- a/docs/reference/pip_download.rst +++ b/docs/reference/pip_download.rst @@ -55,66 +55,66 @@ Options Examples ******** -1. Download a package and all of its dependencies +#. Download a package and all of its dependencies - :: + :: - $ pip download SomePackage - $ pip download -d . SomePackage # equivalent to above - $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + $ pip download SomePackage + $ pip download -d . SomePackage # equivalent to above + $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage -2. Download a package and all of its dependencies with OSX specific interpreter constraints. - This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, - this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, - etc. - It will also match deps with platform ``any``. Also force the interpreter version to ``27`` - (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). +#. Download a package and all of its dependencies with OSX specific interpreter constraints. + This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, + this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, + etc. + It will also match deps with platform ``any``. Also force the interpreter version to ``27`` + (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). - :: + :: - $ pip download \ - --only-binary=:all: \ - --platform macosx-10_10_x86_64 \ - --python-version 27 \ - --implementation cp \ - SomePackage + $ pip download \ + --only-binary=:all: \ + --platform macosx-10_10_x86_64 \ + --python-version 27 \ + --implementation cp \ + SomePackage -3. Download a package and its dependencies with linux specific constraints, including - packages that support the ``manylinux1`` platform. Force the interpreter to be any - minor version of py3k, and only accept ``cp34m`` or ``none`` as the abi. +#. Download a package and its dependencies with linux specific constraints. + Force the interpreter to be any minor version of py3k, and only accept + ``cp34m`` or ``none`` as the abi. - :: + :: - $ pip download \ - --only-binary=:all: \ - --platform linux_x86_64 --manylinux \ - --python-version 3 \ - --implementation cp \ - --abi cp34m \ - SomePackage + $ pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 3 \ + --implementation cp \ + --abi cp34m \ + SomePackage -4. Force platform, implementation, and abi agnostic deps. +#. Force platform, implementation, and abi agnostic deps. - :: + :: - $ pip download \ - --only-binary=:all: \ - --platform any \ - --python-version 3 \ - --implementation py \ - --abi none \ - SomePackage + $ pip download \ + --only-binary=:all: \ + --platform any \ + --python-version 3 \ + --implementation py \ + --abi none \ + SomePackage -5. Even when overconstrained, this will still correctly fetch the pip universal wheel. +#. Even when overconstrained, this will still correctly fetch the pip universal wheel. - :: + :: - $ pip download \ - --only-binary=:all: \ - --platform linux_x86_64 --manylinux \ - --python-version 33 \ - --implementation cp \ - --abi cp34m \ - pip>=8 - $ ls pip-8.1.1-py2.py3-none-any.whl - pip-8.1.1-py2.py3-none-any.whl + $ pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 33 \ + --implementation cp \ + --abi cp34m \ + pip>=8 + $ ls pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl diff --git a/pip/index.py b/pip/index.py index db08b0016..24b97900f 100644 --- a/pip/index.py +++ b/pip/index.py @@ -172,10 +172,6 @@ class PackageFinder(object): abi=abi, impl=implementation, ) - self.valid_tags_noarch = get_supported( - versions=versions, - noarch=True - ) # If we don't have TLS enabled, then WARN if anyplace we're looking # relies on TLS. From 72d9fd11f4f6b2c6ee7df98fd7c6c4c521256c61 Mon Sep 17 00:00:00 2001 From: Patrick Lawson Date: Thu, 21 Jul 2016 13:06:34 -0500 Subject: [PATCH 8/8] Doc nits --- docs/reference/pip_download.rst | 16 ++++++++-------- pip/commands/download.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/reference/pip_download.rst b/docs/reference/pip_download.rst index 466b3e577..153282cce 100644 --- a/docs/reference/pip_download.rst +++ b/docs/reference/pip_download.rst @@ -32,8 +32,8 @@ directory). This directory can later be passed as the value to ``pip install ``pip download`` with the ``--platform``, ``--python-version``, ``--implementation``, and ``--abi`` options provides the ability to fetch dependencies for an interpreter and system other than the ones that pip is -running on. ``--only-binary=:all:`` is required when using any of these -options. It is important to note that these options all default to the +running on. ``--only-binary=:all:`` is required when using any of these +options. It is important to note that these options all default to the current system/interpreter, and not to the most restrictive constraints (e.g. platform any, abi none, etc). To avoid fetching dependencies that happen to match the constraint of the current interpreter (but not your target one), it @@ -55,7 +55,7 @@ Options Examples ******** -#. Download a package and all of its dependencies +#. Download a package and all of its dependencies :: @@ -63,11 +63,11 @@ Examples $ pip download -d . SomePackage # equivalent to above $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage -#. Download a package and all of its dependencies with OSX specific interpreter constraints. +#. Download a package and all of its dependencies with OSX specific interpreter constraints. This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, etc. - It will also match deps with platform ``any``. Also force the interpreter version to ``27`` + It will also match deps with platform ``any``. Also force the interpreter version to ``27`` (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). :: @@ -79,7 +79,7 @@ Examples --implementation cp \ SomePackage -#. Download a package and its dependencies with linux specific constraints. +#. Download a package and its dependencies with linux specific constraints. Force the interpreter to be any minor version of py3k, and only accept ``cp34m`` or ``none`` as the abi. @@ -93,7 +93,7 @@ Examples --abi cp34m \ SomePackage -#. Force platform, implementation, and abi agnostic deps. +#. Force platform, implementation, and abi agnostic deps. :: @@ -105,7 +105,7 @@ Examples --abi none \ SomePackage -#. Even when overconstrained, this will still correctly fetch the pip universal wheel. +#. Even when overconstrained, this will still correctly fetch the pip universal wheel. :: diff --git a/pip/commands/download.py b/pip/commands/download.py index 8e7fbca39..4bc064087 100644 --- a/pip/commands/download.py +++ b/pip/commands/download.py @@ -71,7 +71,7 @@ class DownloadCommand(RequirementCommand): metavar='platform', default=None, help=("Only download wheels compatible with . " - "Defaults to the platform of the local computer."), + "Defaults to the platform of the running system."), ) cmd_opts.add_option(