diff --git a/CHANGES.txt b/CHANGES.txt index 3e14a0e15..be36ef267 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,7 @@ **7.1.0 (unreleased)** +* Allow constraining versions globally without having to know exactly what will + be installed by the pip command. :issue:`2731`. **7.0.3 (2015-06-01)** diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index a687daedf..ac0f170d7 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -120,10 +120,14 @@ For example, to specify :ref:`--no-index <--no-index>` and 2 :ref:`--find-links --find-links http://some.archives.com/archives -Lastly, if you wish, you can refer to other requirements files, like this:: +If you wish, you can refer to other requirements files, like this:: -r more_requirements.txt +You can also refer to constraints files, like this:: + + -c some_constraints.txt + .. _`Requirement Specifiers`: Requirement Specifiers diff --git a/docs/user_guide.rst b/docs/user_guide.rst index a05285102..7ef12cc45 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -110,6 +110,39 @@ See also: `_ +.. _`Constraints Files`: + +Constraints Files +***************** + +Constraints files are requirements files that only control which version of a +requirement is installed, not whether it is installed or not. Their syntax and +contents is nearly identical to :ref:`Requirements Files`. There is one key +difference: Including a package in a constraints file does not trigger +installation of the package. + +Use a constraints file like so: + + :: + + pip install -c constraints.txt + +Constraints files are used for exactly the same reason as requirements files +when you don't know exactly what things you want to install. For instance, say +that the "helloworld" package doesn't work in your environment, so you have a +local patched version. Some things you install depend on "helloworld", and some +don't. + +One way to ensure that the patched version is used consistently is to +manually audit the dependencies of everything you install, and if "helloworld" +is present, write a requirements file to use when installing that thing. + +Constraints files offer a better way: write a single constraints file for your +organisation and use that everywhere. If the thing being installed requires +"helloworld" to be installed, your fixed version specified in your constraints +file will be used. + +Constraints file support was added in pip 7.1. .. _`Installing from Wheels`: diff --git a/pip/basecommand.py b/pip/basecommand.py index 28a076acb..2aa637e91 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -261,6 +261,13 @@ class RequirementCommand(Command): """ Marshal cmd line args into a requirement set. """ + for filename in options.constraints: + for req in parse_requirements( + filename, + constraint=True, finder=finder, options=options, + session=session, wheel_cache=wheel_cache): + requirement_set.add_requirement(req) + for req in args: requirement_set.add_requirement( InstallRequirement.from_line( diff --git a/pip/cmdoptions.py b/pip/cmdoptions.py index ed6486a9e..1836fadf6 100644 --- a/pip/cmdoptions.py +++ b/pip/cmdoptions.py @@ -337,6 +337,17 @@ process_dependency_links = partial( ) +def constraints(): + return Option( + '-c', '--constraint', + dest='constraints', + action='append', + default=[], + metavar='file', + help='Constrain versions using the given constraints file. ' + 'This option can be used multiple times.') + + def requirements(): return Option( '-r', '--requirement', diff --git a/pip/commands/install.py b/pip/commands/install.py index d8fc94bc3..c92bf16ac 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -56,6 +56,7 @@ class InstallCommand(RequirementCommand): cmd_opts = self.cmd_opts + cmd_opts.add_option(cmdoptions.constraints()) cmd_opts.add_option(cmdoptions.editable()) cmd_opts.add_option(cmdoptions.requirements()) cmd_opts.add_option(cmdoptions.build_dir()) diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index 1ad119257..a607fe300 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -69,6 +69,7 @@ class WheelCommand(RequirementCommand): metavar='options', action='append', help="Extra arguments to be supplied to 'setup.py bdist_wheel'.") + cmd_opts.add_option(cmdoptions.constraints()) cmd_opts.add_option(cmdoptions.editable()) cmd_opts.add_option(cmdoptions.requirements()) cmd_opts.add_option(cmdoptions.download_cache()) diff --git a/pip/req/req_file.py b/pip/req/req_file.py index 32b49fc2b..c8811d962 100644 --- a/pip/req/req_file.py +++ b/pip/req/req_file.py @@ -25,6 +25,7 @@ SCHEME_RE = re.compile(r'^(http|https|file):', re.I) COMMENT_RE = re.compile(r'(^|\s)+#.*$') SUPPORTED_OPTIONS = [ + cmdoptions.constraints, cmdoptions.editable, cmdoptions.requirements, cmdoptions.no_index, @@ -54,15 +55,16 @@ SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ] def parse_requirements(filename, finder=None, comes_from=None, options=None, - session=None, wheel_cache=None): - """ - Parse a requirements file and yield InstallRequirement instances. + session=None, constraint=False, wheel_cache=None): + """Parse a requirements file and yield InstallRequirement instances. :param filename: Path or url of requirements file. :param finder: Instance of pip.index.PackageFinder. :param comes_from: Origin description of requirements. :param options: Global options. :param session: Instance of pip.download.PipSession. + :param constraint: If true, parsing a constraint file rather than + requirements file. :param wheel_cache: Instance of pip.wheel.WheelCache """ if session is None: @@ -82,13 +84,15 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, for line_number, line in enumerate(lines, 1): req_iter = process_line(line, filename, line_number, finder, - comes_from, options, session, wheel_cache) + comes_from, options, session, wheel_cache, + constraint=constraint) for req in req_iter: yield req def process_line(line, filename, line_number, finder=None, comes_from=None, - options=None, session=None, wheel_cache=None): + options=None, session=None, wheel_cache=None, + constraint=False): """Process a single requirements line; This can result in creating/yielding requirements, or updating the finder. @@ -103,8 +107,8 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, (although our docs imply only one is supported), and all our parsed and affect the finder. + :param constraint: If True, parsing a constraints file. """ - parser = build_parser() defaults = parser.get_default_values() defaults.index_url = None @@ -114,9 +118,12 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, args_str, options_str = break_args_options(line) opts, _ = parser.parse_args(shlex.split(options_str), defaults) + # preserve for the nested code path + line_comes_from = '%s %s (line %s)' % ( + '-c' if constraint else '-r', filename, line_number) + # yield a line requirement if args_str: - comes_from = '-r %s (line %s)' % (filename, line_number) isolated = options.isolated_mode if options else False if options: cmdoptions.check_install_build_global(options, opts) @@ -126,24 +133,28 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, if dest in opts.__dict__ and opts.__dict__[dest]: req_options[dest] = opts.__dict__[dest] yield InstallRequirement.from_line( - args_str, comes_from, isolated=isolated, options=req_options, - wheel_cache=wheel_cache + args_str, line_comes_from, constraint=constraint, + isolated=isolated, options=req_options, wheel_cache=wheel_cache ) # yield an editable requirement elif opts.editables: - comes_from = '-r %s (line %s)' % (filename, line_number) isolated = options.isolated_mode if options else False default_vcs = options.default_vcs if options else None yield InstallRequirement.from_editable( - opts.editables[0], comes_from=comes_from, - default_vcs=default_vcs, isolated=isolated, + opts.editables[0], comes_from=line_comes_from, + constraint=constraint, default_vcs=default_vcs, isolated=isolated, wheel_cache=wheel_cache ) # parse a nested requirements file - elif opts.requirements: - req_path = opts.requirements[0] + elif opts.requirements or opts.constraints: + if opts.requirements: + req_path = opts.requirements[0] + nested_constraint = False + else: + req_path = opts.constraints[0] + nested_constraint = True # original file is over http if SCHEME_RE.search(filename): # do a url join so relative paths work @@ -156,7 +167,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # TODO: Why not use `comes_from='-r {} (line {})'` here as well? parser = parse_requirements( req_path, finder, comes_from, options, session, - wheel_cache=wheel_cache + constraint=nested_constraint, wheel_cache=wheel_cache ) for req in parser: yield req diff --git a/pip/req/req_install.py b/pip/req/req_install.py index 37af43902..dd716041b 100644 --- a/pip/req/req_install.py +++ b/pip/req/req_install.py @@ -60,7 +60,7 @@ class InstallRequirement(object): def __init__(self, req, comes_from, source_dir=None, editable=False, link=None, as_egg=False, update=True, editable_options=None, pycompile=True, markers=None, isolated=False, options=None, - wheel_cache=None): + wheel_cache=None, constraint=False): self.extras = () if isinstance(req, six.string_types): req = pkg_resources.Requirement.parse(req) @@ -68,6 +68,7 @@ class InstallRequirement(object): self.req = req self.comes_from = comes_from + self.constraint = constraint self.source_dir = source_dir self.editable = editable @@ -106,7 +107,8 @@ class InstallRequirement(object): @classmethod def from_editable(cls, editable_req, comes_from=None, default_vcs=None, - isolated=False, options=None, wheel_cache=None): + isolated=False, options=None, wheel_cache=None, + constraint=False): from pip.index import Link name, url, extras_override, editable_options = parse_editable( @@ -119,6 +121,7 @@ class InstallRequirement(object): res = cls(name, comes_from, source_dir=source_dir, editable=True, link=Link(url), + constraint=constraint, editable_options=editable_options, isolated=isolated, options=options if options else {}, @@ -132,7 +135,7 @@ class InstallRequirement(object): @classmethod def from_line( cls, name, comes_from=None, isolated=False, options=None, - wheel_cache=None): + wheel_cache=None, constraint=False): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ @@ -204,7 +207,7 @@ class InstallRequirement(object): options = options if options else {} res = cls(req, comes_from, link=link, markers=markers, isolated=isolated, options=options, - wheel_cache=wheel_cache) + wheel_cache=wheel_cache, constraint=constraint) if extras: res.extras = pkg_resources.Requirement.parse('__placeholder__' + diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 7b7a44d47..264d99452 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -231,11 +231,16 @@ class RequirementSet(object): self.unnamed_requirements.append(install_req) return [install_req] else: - if parent_req_name is None and self.has_requirement(name): + try: + existing_req = self.get_requirement(name) + except KeyError: + existing_req = None + if (parent_req_name is None and existing_req and not + existing_req.constraint): raise InstallationError( 'Double requirement given: %s (already in %s, name=%r)' - % (install_req, self.get_requirement(name), name)) - if not self.has_requirement(name): + % (install_req, existing_req, name)) + if not existing_req: # Add requirement self.requirements[name] = install_req # FIXME: what about other normalizations? E.g., _ vs. -? @@ -243,10 +248,19 @@ class RequirementSet(object): self.requirement_aliases[name.lower()] = name result = [install_req] else: - # Canonicalise to the already-added object - install_req = self.get_requirement(name) - # No need to scan, this is a duplicate requirement. - result = [] + if not existing_req.constraint: + # No need to scan, we've already encountered this for + # scanning. + result = [] + elif not install_req.constraint: + # If we're now installing a constraint, mark the existing + # object for real installation. + existing_req.constraint = False + # And now we need to scan this. + result = [existing_req] + # Canonicalise to the already-added object for the backref + # check below. + install_req = existing_req if parent_req_name: parent_req = self.get_requirement(parent_req_name) self._dependencies[parent_req].append(install_req) @@ -260,7 +274,8 @@ class RequirementSet(object): @property def has_requirements(self): - return list(self.requirements.values()) or self.unnamed_requirements + return list(req for req in self.requirements.values() if not + req.constraint) or self.unnamed_requirements @property def is_download(self): @@ -285,6 +300,8 @@ class RequirementSet(object): def uninstall(self, auto_confirm=False): for req in self.requirements.values(): + if req.constraint: + continue req.uninstall(auto_confirm=auto_confirm) req.commit_uninstall() @@ -376,6 +393,9 @@ class RequirementSet(object): # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) + if req_to_install.constraint: + return [] + if req_to_install.editable: logger.info('Obtaining %s', req_to_install) else: @@ -584,6 +604,8 @@ class RequirementSet(object): def schedule(req): if req.satisfied_by or req in ordered_reqs: return + if req.constraint: + return ordered_reqs.add(req) for dep in self._dependencies[req]: schedule(dep) diff --git a/pip/wheel.py b/pip/wheel.py index 342c6dbde..e29bb7e2b 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -708,6 +708,8 @@ class WheelBuilder(object): buildset = [] for req in reqset: + if req.constraint: + continue if req.is_wheel: if not autobuilding: logger.info( diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index c227dcf7b..9dbf522ab 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -106,6 +106,16 @@ def test_multiple_requirements_files(script, tmpdir): assert script.venv / 'src' / 'initools' in result.files_created +def test_multiple_constraints_files(script, data): + script.scratch_path.join("outer.txt").write("-c inner.txt") + script.scratch_path.join("inner.txt").write( + "Upper==1.0") + result = script.pip( + 'install', '--no-index', '-f', data.find_links, '-c', + script.scratch_path / 'outer.txt', 'Upper') + assert 'installed Upper-1.0' in result.stdout + + def test_respect_order_in_requirements_file(script, data): script.scratch_path.join("frameworks-req.txt").write(textwrap.dedent("""\ parent @@ -192,3 +202,19 @@ def test_install_option_in_requirements_file(script, data, virtualenv): package_dir = script.scratch / 'home1' / 'lib' / 'python' / 'simple' assert package_dir in result.files_created + + +def test_constraints_not_installed_by_default(script, data): + script.scratch_path.join("c.txt").write("requiresupper") + result = script.pip( + 'install', '--no-index', '-f', data.find_links, '-c', + script.scratch_path / 'c.txt', 'Upper') + assert 'requiresupper' not in result.stdout + + +def test_constraints_only_causes_error(script, data): + script.scratch_path.join("c.txt").write("requiresupper") + result = script.pip( + 'install', '--no-index', '-f', data.find_links, '-c', + script.scratch_path / 'c.txt', expect_error=True) + assert 'installed requiresupper' not in result.stdout diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index ea66e75e3..6a052b26e 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -89,6 +89,16 @@ class TestProcessLine(object): req = InstallRequirement.from_line(line, comes_from=comes_from) assert repr(list(process_line(line, filename, 1))[0]) == repr(req) + def test_yield_line_constraint(self): + line = 'SomeProject' + filename = 'filename' + comes_from = '-c %s (line %s)' % (filename, 1) + req = InstallRequirement.from_line( + line, comes_from=comes_from, constraint=True) + found_req = list(process_line(line, filename, 1, constraint=True))[0] + assert repr(found_req) == repr(req) + assert found_req.constraint is True + def test_yield_line_requirement_with_spaces_in_specifier(self): line = 'SomeProject >= 2' filename = 'filename' @@ -105,18 +115,42 @@ class TestProcessLine(object): req = InstallRequirement.from_editable(url, comes_from=comes_from) assert repr(list(process_line(line, filename, 1))[0]) == repr(req) + def test_yield_editable_constraint(self): + url = 'git+https://url#egg=SomeProject' + line = '-e %s' % url + filename = 'filename' + comes_from = '-c %s (line %s)' % (filename, 1) + req = InstallRequirement.from_editable( + url, comes_from=comes_from, constraint=True) + found_req = list(process_line(line, filename, 1, constraint=True))[0] + assert repr(found_req) == repr(req) + assert found_req.constraint is True + def test_nested_requirements_file(self, monkeypatch): line = '-r another_file' req = InstallRequirement.from_line('SomeProject') import pip.req.req_file def stub_parse_requirements(req_url, finder, comes_from, options, - session, wheel_cache): - return [req] + session, wheel_cache, constraint): + return [(req, constraint)] parse_requirements_stub = stub(call=stub_parse_requirements) monkeypatch.setattr(pip.req.req_file, 'parse_requirements', parse_requirements_stub.call) - assert list(process_line(line, 'filename', 1)) == [req] + assert list(process_line(line, 'filename', 1)) == [(req, False)] + + def test_nested_constraints_file(self, monkeypatch): + line = '-c another_file' + req = InstallRequirement.from_line('SomeProject') + import pip.req.req_file + + def stub_parse_requirements(req_url, finder, comes_from, options, + session, wheel_cache, constraint): + return [(req, constraint)] + parse_requirements_stub = stub(call=stub_parse_requirements) + monkeypatch.setattr(pip.req.req_file, 'parse_requirements', + parse_requirements_stub.call) + assert list(process_line(line, 'filename', 1)) == [(req, True)] def test_options_on_a_requirement_line(self): line = 'SomeProject --install-option=yo1 --install-option yo2 '\ diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 9e4693a04..169592b9d 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -373,7 +373,7 @@ class TestWheelBuilder(object): def test_skip_building_wheels(self, caplog): with patch('pip.wheel.WheelBuilder._build_one') as mock_build_one: - wheel_req = Mock(is_wheel=True, editable=False) + wheel_req = Mock(is_wheel=True, editable=False, constraint=False) reqset = Mock(requirements=Mock(values=lambda: [wheel_req]), wheel_download_dir='/wheel/dir') wb = wheel.WheelBuilder(reqset, Mock()) @@ -383,8 +383,8 @@ class TestWheelBuilder(object): def test_skip_building_editables(self, caplog): with patch('pip.wheel.WheelBuilder._build_one') as mock_build_one: - editable_req = Mock(editable=True, is_wheel=False) - reqset = Mock(requirements=Mock(values=lambda: [editable_req]), + editable = Mock(editable=True, is_wheel=False, constraint=False) + reqset = Mock(requirements=Mock(values=lambda: [editable]), wheel_download_dir='/wheel/dir') wb = wheel.WheelBuilder(reqset, Mock()) wb.build()