McSinyx/pip
1
1
Fork 0
mirror of https://github.com/pypa/pip synced 2023-12-13 21:30:23 +01:00
Code Issues Releases Activity

Merge pull request #3760 from patricklaw/pl.pip-install-additional-options

Browse source 
Pl.pip install additional options
This commit is contained in:
Xavier Fernandez 2016-08-12 10:27:57 +02:00 committed by GitHub
parent b506992fcc 72d9fd11f4
commit 5bd3367644
8 changed files with 606 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
CHANGES.txt
View file

@ -18,6 +18,11 @@
editable dependencies before); this allows running pip wheel on the result
of pip freeze in presence of editable requirements (:issue:`3291`)
* Add --platform, --python-version, --implementation and --abi parameters to
``pip download``. These allow utilities and advanced users to gather
distributions for interpreters other than the one pip is being run on.
(:pull:`3760`)
**8.1.2 (2016-05-10)**

75
docs/reference/pip_download.rst
View file

@ -26,9 +26,22 @@ which is now deprecated and will be removed in pip 10.
``pip download`` does the same resolution and downloading as ``pip install``,
but instead of installing the dependencies, it collects the downloaded
distributions into the directory provided (defaulting to the current
directory). This directory can later be passed as the value to
``pip install --find-links`` to facilitate offline or locked down package
installation.
directory). This directory can later be passed as the value to ``pip install
--find-links`` to facilitate offline or locked down package installation.
``pip download`` with the ``--platform``, ``--python-version``,
``--implementation``, and ``--abi`` options provides the ability to fetch
dependencies for an interpreter and system other than the ones that pip is
running on. ``--only-binary=:all:`` is required when using any of these
options. It is important to note that these options all default to the
current system/interpreter, and not to the most restrictive constraints (e.g.
platform any, abi none, etc). To avoid fetching dependencies that happen to
match the constraint of the current interpreter (but not your target one), it
is recommended to specify all of these options if you are specifying one of
them. Generic dependencies (e.g. universal wheels, or dependencies with no
platform, abi, or implementation constraints) will still match an over-
constrained download requirement.
Options
@ -49,3 +62,59 @@ Examples
$ pip download SomePackage
$ pip download -d . SomePackage # equivalent to above
$ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage
#. Download a package and all of its dependencies with OSX specific interpreter constraints.
This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible,
this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``,
etc.
It will also match deps with platform ``any``. Also force the interpreter version to ``27``
(or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``).
::
$ pip download \
--only-binary=:all: \
--platform macosx-10_10_x86_64 \
--python-version 27 \
--implementation cp \
SomePackage
#. Download a package and its dependencies with linux specific constraints.
Force the interpreter to be any minor version of py3k, and only accept
``cp34m`` or ``none`` as the abi.
::
$ pip download \
--only-binary=:all: \
--platform linux_x86_64 \
--python-version 3 \
--implementation cp \
--abi cp34m \
SomePackage
#. Force platform, implementation, and abi agnostic deps.
::
$ pip download \
--only-binary=:all: \
--platform any \
--python-version 3 \
--implementation py \
--abi none \
SomePackage
#. Even when overconstrained, this will still correctly fetch the pip universal wheel.
::
$ pip download \
--only-binary=:all: \
--platform linux_x86_64 \
--python-version 33 \
--implementation cp \
--abi cp34m \
pip>=8
$ ls pip-8.1.1-py2.py3-none-any.whl
pip-8.1.1-py2.py3-none-any.whl

8
pip/basecommand.py
View file

@ -311,7 +311,9 @@ class RequirementCommand(Command):
'to %(name)s (see "pip help %(name)s")' % opts)
logger.warning(msg)
def _build_package_finder(self, options, session):
def _build_package_finder(self, options, session,
platform=None, python_versions=None,
abi=None, implementation=None):
"""
Create a package finder appropriate to this requirement command.
"""
@ -328,4 +330,8 @@ class RequirementCommand(Command):
allow_all_prereleases=options.pre,
process_dependency_links=options.process_dependency_links,
session=session,
platform=platform,
versions=python_versions,
abi=abi,
implementation=implementation,
)

80
pip/commands/download.py
View file

@ -3,6 +3,8 @@ from __future__ import absolute_import
import logging
import os
from pip.exceptions import CommandError
from pip.index import FormatControl
from pip.req import RequirementSet
from pip.basecommand import RequirementCommand
from pip import cmdoptions
@ -63,6 +65,53 @@ class DownloadCommand(RequirementCommand):
help=("Download packages into <dir>."),
)
cmd_opts.add_option(
'--platform',
dest='platform',
metavar='platform',
default=None,
help=("Only download wheels compatible with <platform>. "
"Defaults to the platform of the running system."),
)
cmd_opts.add_option(
'--python-version',
dest='python_version',
metavar='python_version',
default=None,
help=("Only download wheels compatible with Python "
"interpreter version <version>. If not specified, then the "
"current system interpreter minor version is used. A major "
"version (e.g. '2') can be specified to match all "
"minor revs of that major version. A minor version "
"(e.g. '34') can also be specified."),
)
cmd_opts.add_option(
'--implementation',
dest='implementation',
metavar='implementation',
default=None,
help=("Only download wheels compatible with Python "
"implementation <implementation>, e.g. 'pp', 'jy', 'cp', "
" or 'ip'. If not specified, then the current "
"interpreter implementation is used. Use 'py' to force "
"implementation-agnostic wheels."),
)
cmd_opts.add_option(
'--abi',
dest='abi',
metavar='abi',
default=None,
help=("Only download wheels compatible with Python "
"abi <abi>, e.g. 'pypy_41'. If not specified, then the "
"current interpreter abi tag is used. Generally "
"you will need to specify --implementation, "
"--platform, and --python-version when using "
"this option."),
)
index_opts = cmdoptions.make_option_group(
cmdoptions.non_deprecated_index_group,
self.parser,
@ -73,14 +122,41 @@ class DownloadCommand(RequirementCommand):
def run(self, options, args):
options.ignore_installed = True
if options.python_version:
python_versions = [options.python_version]
else:
python_versions = None
dist_restriction_set = any([
options.python_version,
options.platform,
options.abi,
options.implementation,
])
binary_only = FormatControl(set(), set([':all:']))
if dist_restriction_set and options.format_control != binary_only:
raise CommandError(
"--only-binary=:all: must be set and --no-binary must not "
"be set (or must be set to :none:) when restricting platform "
"and interpreter constraints using --python-version, "
"--platform, --abi, or --implementation."
)
options.src_dir = os.path.abspath(options.src_dir)
options.download_dir = normalize_path(options.download_dir)
ensure_dir(options.download_dir)
with self._build_session(options) as session:
finder = self._build_package_finder(options, session)
finder = self._build_package_finder(
options=options,
session=session,
platform=options.platform,
python_versions=python_versions,
abi=options.abi,
implementation=options.implementation,
)
build_delete = (not (options.no_clean or options.build_dir))
if options.cache_dir and not check_path_owner(options.cache_dir):
logger.warning(

36
pip/index.py
View file

@ -29,7 +29,7 @@ from pip.exceptions import (
)
from pip.download import HAS_TLS, is_url, path_to_url, url_to_path
from pip.wheel import Wheel, wheel_ext
from pip.pep425tags import supported_tags
from pip.pep425tags import get_supported
from pip._vendor import html5lib, requests, six
from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.packaging.utils import canonicalize_name
@ -107,12 +107,24 @@ class PackageFinder(object):
def __init__(self, find_links, index_urls, allow_all_prereleases=False,
trusted_hosts=None, process_dependency_links=False,
session=None, format_control=None):
session=None, format_control=None, platform=None,
versions=None, abi=None, implementation=None):
"""Create a PackageFinder.
:param format_control: A FormatControl object or None. Used to control
the selection of source packages / binary packages when consulting
the index and links.
:param platform: A string or None. If None, searches for packages
that are supported by the current system. Otherwise, will find
packages that can be built on the platform passed in. These
packages will only be downloaded for distribution: they will
not be built locally.
:param versions: A list of strings or None. This is passed directly
to pep425tags.py in the get_supported() method.
:param abi: A string or None. This is passed directly
to pep425tags.py in the get_supported() method.
:param implementation: A string or None. This is passed directly
to pep425tags.py in the get_supported() method.
"""
if session is None:
raise TypeError(
@ -156,6 +168,14 @@ class PackageFinder(object):
# The Session we'll use to make requests
self.session = session
# The valid tags to check potential found wheel candidates against
self.valid_tags = get_supported(
versions=versions,
platform=platform,
abi=abi,
impl=implementation,
)
# If we don't have TLS enabled, then WARN if anyplace we're looking
# relies on TLS.
if not HAS_TLS:
@ -239,22 +259,22 @@ class PackageFinder(object):
If not finding wheels, then sorted by version only.
If finding wheels, then the sort order is by version, then:
1. existing installs
2. wheels ordered via Wheel.support_index_min()
2. wheels ordered via Wheel.support_index_min(self.valid_tags)
3. source archives
Note: it was considered to embed this logic into the Link
comparison operators, but then different sdist links
with the same version, would have to be considered equal
"""
support_num = len(supported_tags)
support_num = len(self.valid_tags)
if candidate.location.is_wheel:
# can raise InvalidWheelFilename
wheel = Wheel(candidate.location.filename)
if not wheel.supported():
if not wheel.supported(self.valid_tags):
raise UnsupportedWheel(
"%s is not a supported wheel for this platform. It "
"can't be sorted." % wheel.filename
)
pri = -(wheel.support_index_min())
pri = -(wheel.support_index_min(self.valid_tags))
else: # sdist
pri = -(support_num)
return (candidate.version, pri)
@ -584,7 +604,6 @@ class PackageFinder(object):
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
@ -615,7 +634,8 @@ class PackageFinder(object):
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported():
if not wheel.supported(self.valid_tags):
self._log_skipped_link(
link, 'it is not compatible with this Python')
return

19
pip/pep425tags.py
View file

@ -223,12 +223,19 @@ def get_darwin_arches(major, minor, machine):
return arches
def get_supported(versions=None, noarch=False):
def get_supported(versions=None, noarch=False, platform=None,
impl=None, abi=None):
"""Return a list of supported tags for each version specified in
`versions`.
:param versions: a list of string versions, of the form ["33", "32"],
or None. The first version will be assumed to support our ABI.
:param platform: specify the exact platform you want valid
tags for, or None. If None, use the local system platform.
:param impl: specify the exact implementation you want valid
tags for, or None. If None, use the local interpreter impl.
:param abi: specify the exact abi you want valid
tags for, or None. If None, use the local interpreter abi.
"""
supported = []
@ -241,11 +248,11 @@ def get_supported(versions=None, noarch=False):
for minor in range(version_info[-1], -1, -1):
versions.append(''.join(map(str, major + (minor,))))
impl = get_abbr_impl()
impl = impl or get_abbr_impl()
abis = []
abi = get_abi_tag()
abi = abi or get_abi_tag()
if abi:
abis[0:0] = [abi]
@ -260,8 +267,8 @@ def get_supported(versions=None, noarch=False):
abis.append('none')
if not noarch:
arch = get_platform()
if sys.platform == 'darwin':
arch = platform or get_platform()
if arch.startswith('macosx'):
# support macosx-10.6-intel on macosx-10.9-x86_64
match = _osx_arch_pat.match(arch)
if match:
@ -274,7 +281,7 @@ def get_supported(versions=None, noarch=False):
else:
# arch pattern didn't match (?!)
arches = [arch]
elif is_manylinux1_compatible():
elif platform is None and is_manylinux1_compatible():
arches = [arch.replace('linux', 'manylinux1'), arch]
else:
arches = [arch]

397
tests/functional/test_download.py
View file

@ -5,6 +5,12 @@ import pytest
from tests.lib.path import Path
def fake_wheel(data, wheel_path):
data.packages.join(
'simple.dist-0.1-py2.py3-none-any.whl'
).copy(data.packages.join(wheel_path))
@pytest.mark.network
def test_download_if_requested(script):
"""
@ -157,3 +163,394 @@ def test_download_vcs_link(script):
in result.files_created
)
assert script.site_packages / 'piptestpackage' not in result.files_created
def test_download_specify_platform_only_binary(script, data):
"""
Confirm that specifying an interpreter/platform constraint
enforces that ``--only-binary=:all:`` is set.
"""
fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'linux_x86_64',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-py2.py3-none-any.whl'
in result.files_created
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--dest', '.',
'--platform', 'linux_x86_64',
'fake',
expect_error=True,
)
assert '--only-binary=:all:' in result.stderr
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--dest', '.',
'--platform', 'linux_x86_64',
'fake',
expect_error=True,
)
assert '--only-binary=:all:' in result.stderr
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--no-binary=fake',
'--dest', '.',
'--platform', 'linux_x86_64',
'fake',
expect_error=True,
)
assert '--only-binary=:all:' in result.stderr
def test_download_specify_platform(script, data):
"""
Test using "pip download --platform" to download a .whl archive
supported for a specific platform
"""
fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl')
# Confirm that universal wheels are returned even for specific
# platforms.
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'linux_x86_64',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-py2.py3-none-any.whl'
in result.files_created
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'macosx_10_9_x86_64',
'fake'
)
data.reset()
fake_wheel(data, 'fake-1.0-py2.py3-none-macosx_10_9_x86_64.whl')
fake_wheel(data, 'fake-2.0-py2.py3-none-linux_x86_64.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'macosx_10_10_x86_64',
'fake'
)
assert (
Path('scratch') /
'fake-1.0-py2.py3-none-macosx_10_9_x86_64.whl'
in result.files_created
)
# OSX platform wheels are not backward-compatible.
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'macosx_10_8_x86_64',
'fake',
expect_error=True,
)
# No linux wheel provided for this version.
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'linux_x86_64',
'fake==1',
expect_error=True,
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'linux_x86_64',
'fake==2'
)
assert (
Path('scratch') / 'fake-2.0-py2.py3-none-linux_x86_64.whl'
in result.files_created
)
def test_download_platform_manylinux(script, data):
"""
Test using "pip download --platform" to download a .whl archive
supported for a specific platform.
"""
fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl')
# Confirm that universal wheels are returned even for specific
# platforms.
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'linux_x86_64',
'fake',
)
assert (
Path('scratch') / 'fake-1.0-py2.py3-none-any.whl'
in result.files_created
)
data.reset()
fake_wheel(data, 'fake-1.0-py2.py3-none-manylinux1_x86_64.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'manylinux1_x86_64',
'fake',
)
assert (
Path('scratch') /
'fake-1.0-py2.py3-none-manylinux1_x86_64.whl'
in result.files_created
)
# When specifying the platform, manylinux1 needs to be the
# explicit platform--it won't ever be added to the compatible
# tags.
data.reset()
fake_wheel(data, 'fake-1.0-py2.py3-none-linux_x86_64.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--platform', 'linux_x86_64',
'fake',
expect_error=True,
)
def test_download_specify_python_version(script, data):
"""
Test using "pip download --python-version" to download a .whl archive
supported for a specific interpreter
"""
fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '2',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-py2.py3-none-any.whl'
in result.files_created
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '3',
'fake'
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '27',
'fake'
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '33',
'fake'
)
data.reset()
fake_wheel(data, 'fake-1.0-py2-none-any.whl')
fake_wheel(data, 'fake-2.0-py3-none-any.whl')
# No py3 provided for version 1.
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '3',
'fake==1.0',
expect_error=True,
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '2',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-py2-none-any.whl'
in result.files_created
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '26',
'fake'
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '3',
'fake'
)
assert (
Path('scratch') / 'fake-2.0-py3-none-any.whl'
in result.files_created
)
def test_download_specify_abi(script, data):
"""
Test using "pip download --abi" to download a .whl archive
supported for a specific abi
"""
fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--implementation', 'fk',
'--abi', 'fake_abi',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-py2.py3-none-any.whl'
in result.files_created
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--implementation', 'fk',
'--abi', 'none',
'fake'
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--abi', 'cp27m',
'fake',
expect_error=True,
)
data.reset()
fake_wheel(data, 'fake-1.0-fk2-fakeabi-fake_platform.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--python-version', '2',
'--implementation', 'fk',
'--platform', 'fake_platform',
'--abi', 'fakeabi',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-fk2-fakeabi-fake_platform.whl'
in result.files_created
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--implementation', 'fk',
'--platform', 'fake_platform',
'--abi', 'none',
'fake',
expect_error=True,
)
def test_download_specify_implementation(script, data):
"""
Test using "pip download --abi" to download a .whl archive
supported for a specific abi
"""
fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--implementation', 'fk',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-py2.py3-none-any.whl'
in result.files_created
)
data.reset()
fake_wheel(data, 'fake-1.0-fk2.fk3-none-any.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--implementation', 'fk',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-fk2.fk3-none-any.whl'
in result.files_created
)
data.reset()
fake_wheel(data, 'fake-1.0-fk3-none-any.whl')
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--implementation', 'fk',
'--python-version', '3',
'fake'
)
assert (
Path('scratch') / 'fake-1.0-fk3-none-any.whl'
in result.files_created
)
result = script.pip(
'download', '--no-index', '--find-links', data.find_links,
'--only-binary=:all:',
'--dest', '.',
'--implementation', 'fk',
'--python-version', '2',
'fake',
expect_error=True,
)

13
tests/unit/test_finder.py
View file

@ -150,6 +150,7 @@ class TestWheel:
[],
session=PipSession(),
)
finder.valid_tags = pip.pep425tags.supported_tags
with pytest.raises(DistributionNotFound):
finder.find_requirement(req, True)
@ -211,11 +212,6 @@ class TestWheel:
with pytest.raises(BestVersionAlreadyInstalled):
finder.find_requirement(req, True)
@patch('pip.pep425tags.supported_tags', [
('pyT', 'none', 'TEST'),
('pyT', 'TEST', 'any'),
('pyT', 'none', 'any'),
])
def test_link_sorting(self):
"""
Test link sorting
@ -243,9 +239,12 @@ class TestWheel:
Link('simple-1.0.tar.gz'),
),
]
finder = PackageFinder([], [], session=PipSession())
finder.valid_tags = [
('pyT', 'none', 'TEST'),
('pyT', 'TEST', 'any'),
('pyT', 'none', 'any'),
]
results = sorted(links,
key=finder._candidate_sort_key, reverse=True)
results2 = sorted(reversed(links),