""" Requirements file parsing """ from __future__ import absolute_import import os import re import shlex import optparse from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._vendor.six.moves import filterfalse import pip from pip.download import get_file_content from pip.req.req_install import InstallRequirement from pip.exceptions import (RequirementsFileParseError, ReqFileOnleOneOptionPerLineError, ReqFileOptionNotAllowedWithReqError) from pip.utils import normalize_name from pip import cmdoptions __all__ = ['parse_requirements'] SCHEME_RE = re.compile(r'^(http|https|file):', re.I) COMMENT_RE = re.compile(r'(^|\s)+#.*$') SUPPORTED_OPTIONS = [ cmdoptions.editable, cmdoptions.requirements, cmdoptions.no_index, cmdoptions.index_url, cmdoptions.find_links, cmdoptions.extra_index_url, cmdoptions.allow_external, cmdoptions.no_allow_external, cmdoptions.allow_unsafe, cmdoptions.no_allow_unsafe, cmdoptions.use_wheel, cmdoptions.no_use_wheel, cmdoptions.always_unzip, ] # options allowed on requirement lines SUPPORTED_OPTIONS_REQ = [ cmdoptions.install_options, cmdoptions.global_options, cmdoptions.no_binary, cmdoptions.only_binary, ] # the 'dest' string values SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ] def parse_requirements(filename, finder=None, comes_from=None, options=None, session=None, wheel_cache=None): """ Parse a requirements file and yield InstallRequirement instances. :param filename: Path or url of requirements file. :param finder: Instance of pip.index.PackageFinder. :param comes_from: Origin description of requirements. :param options: Global options. :param session: Instance of pip.download.PipSession. :param wheel_cache: Instance of pip.wheel.WheelCache """ if session is None: raise TypeError( "parse_requirements() missing 1 required keyword argument: " "'session'" ) _, content = get_file_content( filename, comes_from=comes_from, session=session ) lines = content.splitlines() lines = ignore_comments(lines) lines = join_lines(lines) lines = skip_regex(lines, options) for line_number, line in enumerate(lines, 1): req_iter = process_line(line, filename, line_number, finder, comes_from, options, session, wheel_cache) for req in req_iter: yield req def process_line(line, filename, line_number, finder=None, comes_from=None, options=None, session=None, wheel_cache=None): """ Process a single requirements line; This can result in creating/yielding requirements, or updating the finder. """ parser = build_parser() values = parser.get_default_values() if finder: values.format_control = finder.format_control else: # Undo the hack that removes defaults so that # this can be parsed correctly. values.format_control = pip.index.FormatControl(set(), set()) orig_no_binary = frozenset(values.format_control.no_binary) orig_only_binary = frozenset(values.format_control.only_binary) args = shlex.split(line) opts, args = parser.parse_args(args, values) if opts.use_wheel is False and finder: pip.index.fmt_ctl_no_use_wheel(finder.format_control) setattr(values, 'use_wheel', None) if (orig_no_binary == opts.format_control.no_binary and orig_only_binary == opts.format_control.only_binary): # Make the per-requirement-line check work. setattr(values, 'format_control', None) req = None if args: for key, value in opts.__dict__.items(): # only certain options can be on req lines if value is not None and key not in SUPPORTED_OPTIONS_REQ_DEST: # get the option string # the option must be supported to get to this point for o in SUPPORTED_OPTIONS: o = o() if o.dest == key: opt_string = o.get_opt_string() msg = ('Option not supported on a' ' requirement line: %s' % opt_string) raise ReqFileOptionNotAllowedWithReqError(msg) # don't allow multiple/different options (on non-req lines) if not args and len( [v for v in opts.__dict__.values() if v is not None]) > 1: msg = 'Only one option allowed per line.' raise ReqFileOnleOneOptionPerLineError(msg) # yield a line requirement if args: args_line = ' '.join(args) comes_from = '-r %s (line %s)' % (filename, line_number) isolated = options.isolated_mode if options else False # trim the None items keys = [opt for opt in opts.__dict__ if getattr(opts, opt) is None] for key in keys: delattr(opts, key) yield InstallRequirement.from_line( args_line, comes_from, isolated=isolated, options=opts.__dict__, wheel_cache=wheel_cache ) # yield an editable requirement elif opts.editables: comes_from = '-r %s (line %s)' % (filename, line_number) isolated = options.isolated_mode if options else False default_vcs = options.default_vcs if options else None yield InstallRequirement.from_editable( opts.editables[0], comes_from=comes_from, default_vcs=default_vcs, isolated=isolated, wheel_cache=wheel_cache ) # parse a nested requirements file elif opts.requirements: req_file = opts.requirements[0] if SCHEME_RE.search(filename): # Relative to an URL. req_url = urllib_parse.urljoin(filename, req_file) elif not SCHEME_RE.search(req_file): req_dir = os.path.dirname(filename) req_url = os.path.join(os.path.dirname(filename), req_file) # TODO: Why not use `comes_from='-r {} (line {})'` here as well? parser = parse_requirements( req_url, finder, comes_from, options, session, wheel_cache=wheel_cache ) for req in parser: yield req # set finder options elif finder: if opts.use_wheel is not None: finder.use_wheel = opts.use_wheel elif opts.no_index is not None: finder.index_urls = [] elif opts.allow_all_external is not None: finder.allow_all_external = opts.allow_all_external elif opts.index_url is not None: finder.index_urls = [opts.index_url] elif opts.extra_index_urls is not None: finder.index_urls.extend(opts.extra_index_urls) elif opts.allow_external is not None: finder.allow_external |= set( [normalize_name(v).lower() for v in opts.allow_external]) elif opts.allow_unverified is not None: # Remove after 7.0 finder.allow_unverified |= set( [normalize_name(v).lower() for v in opts.allow_unverified]) elif opts.find_links is not None: # FIXME: it would be nice to keep track of the source # of the find_links: support a find-links local path # relative to a requirements file. value = opts.find_links[0] req_dir = os.path.dirname(os.path.abspath(filename)) relative_to_reqs_file = os.path.join(req_dir, value) if os.path.exists(relative_to_reqs_file): value = relative_to_reqs_file finder.find_links.append(value) def build_parser(): """ Return a parser for parsing requirement lines """ parser = optparse.OptionParser(add_help_option=False) options = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ for option in options: option = option() # we want no default values; defaults are handled in `pip install` # parsing. just concerned with values that are specifically set. option.default = None parser.add_option(option) # By default optparse sys.exits on parsing errors. We want to wrap # that in our own exception. def parser_exit(self, msg): raise RequirementsFileParseError(msg) parser.exit = parser_exit return parser def join_lines(iterator): """ Joins a line ending in '\' with the previous line. """ lines = [] for line in iterator: if not line.endswith('\\'): if lines: lines.append(line) yield ''.join(lines) lines = [] else: yield line else: lines.append(line.strip('\\')) # TODO: handle space after '\'. # TODO: handle '\' on last line. def ignore_comments(iterator): """ Strips and filters empty or commented lines. """ for line in iterator: line = COMMENT_RE.sub('', line) line = line.strip() if line: yield line def skip_regex(lines, options): """ Optionally exclude lines that match '--skip-requirements-regex' """ skip_regex = options.skip_requirements_regex if options else None if skip_regex: lines = filterfalse(re.compile(skip_regex).search, lines) return lines