From 4d5c5f8f977d55f84ac6d7ea97c213ed91ca081d Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 9 Mar 2013 09:19:15 -0500 Subject: [PATCH] Only install stable releases by default * Adds the --pre flag to pip install to specify that you wish to install pre-releases * If the version spec contains any pre-releases then continue to install pre-releases --- pip/commands/install.py | 8 +- pip/distlib/__init__.py | 1 + pip/distlib/version.py | 222 ++++++++++++++++++++++++++++++++++++++++ pip/index.py | 5 +- pip/req.py | 20 ++-- pip/util.py | 18 ++++ setup.py | 2 +- 7 files changed, 267 insertions(+), 9 deletions(-) create mode 100644 pip/distlib/__init__.py create mode 100644 pip/distlib/version.py diff --git a/pip/commands/install.py b/pip/commands/install.py index 2de43453f..74de46f73 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -177,6 +177,12 @@ class InstallCommand(Command): default=None, help="Install everything relative to this alternate root directory.") + cmd_opts.add_option( + '--pre', + action='store_true', + default=False, + help="Include pre-releases in the available versions.") + index_opts = make_option_group(index_group, self.parser) self.parser.insert_option_group(0, index_opts) @@ -232,7 +238,7 @@ class InstallCommand(Command): use_user_site=options.use_user_site) for name in args: requirement_set.add_requirement( - InstallRequirement.from_line(name, None)) + InstallRequirement.from_line(name, None, prereleases=options.pre)) for name in options.editables: requirement_set.add_requirement( InstallRequirement.from_editable(name, default_vcs=options.default_vcs)) diff --git a/pip/distlib/__init__.py b/pip/distlib/__init__.py new file mode 100644 index 000000000..ec396862b --- /dev/null +++ b/pip/distlib/__init__.py @@ -0,0 +1 @@ +from . import version diff --git a/pip/distlib/version.py b/pip/distlib/version.py new file mode 100644 index 000000000..54b123a9b --- /dev/null +++ b/pip/distlib/version.py @@ -0,0 +1,222 @@ +# Taken from distlib so we can easily determine what a pre-release is. +# It has been stripped down to only the parts needed for that purpose. +import re + + +class UnsupportedVersionError(Exception): + """This is an unsupported version.""" + pass + + +class HugeMajorVersionError(UnsupportedVersionError): + """An irrational version because the major version number is huge + (often because a year or date was used). + + See `error_on_huge_major_num` option in `NormalizedVersion` for details. + This guard can be disabled by setting that option False. + """ + pass + + +# A marker used in the second and third parts of the `parts` tuple, for +# versions that don't have those segments, to sort properly. An example +# of versions in sort order ('highest' last): +# 1.0b1 ((1,0), ('b',1), ('z',)) +# 1.0.dev345 ((1,0), ('z',), ('dev', 345)) +# 1.0 ((1,0), ('z',), ('z',)) +# 1.0.post256.dev345 ((1,0), ('z',), ('z', 'post', 256, 'dev', 345)) +# 1.0.post345 ((1,0), ('z',), ('z', 'post', 345, 'z')) +# ^ ^ ^ +# 'b' < 'z' ---------------------/ | | +# | | +# 'dev' < 'z' ----------------------------/ | +# | +# 'dev' < 'z' ----------------------------------------------/ +# 'f' for 'final' would be kind of nice, but due to bugs in the support of +# 'rc' we must use 'z' +_FINAL_MARKER = ('z',) + +_VERSION_RE = re.compile(r''' + ^ + (?P\d+\.\d+(\.\d+)*) # minimum 'N.N' + (?: + (?P[abc]|rc) # 'a'=alpha, 'b'=beta, 'c'=release candidate + # 'rc'= alias for release candidate + (?P\d+(?:\.\d+)*) + )? + (?P(\.post(?P\d+))?(\.dev(?P\d+))?)? + $''', re.VERBOSE) + + +def _parse_numdots(s, full_ver, drop_zeroes=False, min_length=0): + """Parse 'N.N.N' sequences, return a list of ints. + + @param s {str} 'N.N.N...' sequence to be parsed + @param full_ver_str {str} The full version string from which this + comes. Used for error strings. + @param min_length {int} The length to which to pad the + returned list with zeros, if necessary. Default 0. + """ + result = [] + for n in s.split("."): + #if len(n) > 1 and n[0] == '0': + # raise UnsupportedVersionError("cannot have leading zero in " + # "version number segment: '%s' in %r" % (n, full_ver)) + result.append(int(n)) + if drop_zeroes: + while (result and result[-1] == 0 and + (1 + len(result)) > min_length): + result.pop() + return result + + +def normalized_key(s, fail_on_huge_major_ver=True): + match = _VERSION_RE.search(s) + if not match: + raise UnsupportedVersionError(s) + + groups = match.groupdict() + parts = [] + + # main version + block = _parse_numdots(groups['version'], s, min_length=2) + parts.append(tuple(block)) + + # prerelease + prerel = groups.get('prerel') + if prerel is not None: + block = [prerel] + block += _parse_numdots(groups.get('prerelversion'), s, min_length=1) + parts.append(tuple(block)) + else: + parts.append(_FINAL_MARKER) + + # postdev + if groups.get('postdev'): + post = groups.get('post') + dev = groups.get('dev') + postdev = [] + if post is not None: + postdev.extend((_FINAL_MARKER[0], 'post', int(post))) + if dev is None: + postdev.append(_FINAL_MARKER[0]) + if dev is not None: + postdev.extend(('dev', int(dev))) + parts.append(tuple(postdev)) + else: + parts.append(_FINAL_MARKER) + if fail_on_huge_major_ver and parts[0][0] > 1980: + raise HugeMajorVersionError("huge major version number, %r, " + "which might cause future problems: %r" % (parts[0][0], s)) + return tuple(parts) + + +def suggest_normalized_version(s): + """Suggest a normalized version close to the given version string. + + If you have a version string that isn't rational (i.e. NormalizedVersion + doesn't like it) then you might be able to get an equivalent (or close) + rational version from this function. + + This does a number of simple normalizations to the given string, based + on observation of versions currently in use on PyPI. Given a dump of + those version during PyCon 2009, 4287 of them: + - 2312 (53.93%) match NormalizedVersion without change + with the automatic suggestion + - 3474 (81.04%) match when using this suggestion method + + @param s {str} An irrational version string. + @returns A rational version string, or None, if couldn't determine one. + """ + try: + normalized_key(s) + return s # already rational + except UnsupportedVersionError: + pass + + rs = s.lower() + + # part of this could use maketrans + for orig, repl in (('-alpha', 'a'), ('-beta', 'b'), ('alpha', 'a'), + ('beta', 'b'), ('rc', 'c'), ('-final', ''), + ('-pre', 'c'), + ('-release', ''), ('.release', ''), ('-stable', ''), + ('+', '.'), ('_', '.'), (' ', ''), ('.final', ''), + ('final', '')): + rs = rs.replace(orig, repl) + + # if something ends with dev or pre, we add a 0 + rs = re.sub(r"pre$", r"pre0", rs) + rs = re.sub(r"dev$", r"dev0", rs) + + # if we have something like "b-2" or "a.2" at the end of the + # version, that is pobably beta, alpha, etc + # let's remove the dash or dot + rs = re.sub(r"([abc]|rc)[\-\.](\d+)$", r"\1\2", rs) + + # 1.0-dev-r371 -> 1.0.dev371 + # 0.1-dev-r79 -> 0.1.dev79 + rs = re.sub(r"[\-\.](dev)[\-\.]?r?(\d+)$", r".\1\2", rs) + + # Clean: 2.0.a.3, 2.0.b1, 0.9.0~c1 + rs = re.sub(r"[.~]?([abc])\.?", r"\1", rs) + + # Clean: v0.3, v1.0 + if rs.startswith('v'): + rs = rs[1:] + + # Clean leading '0's on numbers. + #TODO: unintended side-effect on, e.g., "2003.05.09" + # PyPI stats: 77 (~2%) better + rs = re.sub(r"\b0+(\d+)(?!\d)", r"\1", rs) + + # Clean a/b/c with no version. E.g. "1.0a" -> "1.0a0". Setuptools infers + # zero. + # PyPI stats: 245 (7.56%) better + rs = re.sub(r"(\d+[abc])$", r"\g<1>0", rs) + + # the 'dev-rNNN' tag is a dev tag + rs = re.sub(r"\.?(dev-r|dev\.r)\.?(\d+)$", r".dev\2", rs) + + # clean the - when used as a pre delimiter + rs = re.sub(r"-(a|b|c)(\d+)$", r"\1\2", rs) + + # a terminal "dev" or "devel" can be changed into ".dev0" + rs = re.sub(r"[\.\-](dev|devel)$", r".dev0", rs) + + # a terminal "dev" can be changed into ".dev0" + rs = re.sub(r"(?![\.\-])dev$", r".dev0", rs) + + # a terminal "final" or "stable" can be removed + rs = re.sub(r"(final|stable)$", "", rs) + + # The 'r' and the '-' tags are post release tags + # 0.4a1.r10 -> 0.4a1.post10 + # 0.9.33-17222 -> 0.9.33.post17222 + # 0.9.33-r17222 -> 0.9.33.post17222 + rs = re.sub(r"\.?(r|-|-r)\.?(\d+)$", r".post\2", rs) + + # Clean 'r' instead of 'dev' usage: + # 0.9.33+r17222 -> 0.9.33.dev17222 + # 1.0dev123 -> 1.0.dev123 + # 1.0.git123 -> 1.0.dev123 + # 1.0.bzr123 -> 1.0.dev123 + # 0.1a0dev.123 -> 0.1a0.dev123 + # PyPI stats: ~150 (~4%) better + rs = re.sub(r"\.?(dev|git|bzr)\.?(\d+)$", r".dev\2", rs) + + # Clean '.pre' (normalized from '-pre' above) instead of 'c' usage: + # 0.2.pre1 -> 0.2c1 + # 0.2-c1 -> 0.2c1 + # 1.0preview123 -> 1.0c123 + # PyPI stats: ~21 (0.62%) better + rs = re.sub(r"\.?(pre|preview|-c)(\d+)$", r"c\g<2>", rs) + + # Tcl/Tk uses "px" for their post release markers + rs = re.sub(r"p(\d+)$", r".post\1", rs) + + try: + normalized_key(rs) + except UnsupportedVersionError: + rs = None + return rs diff --git a/pip/index.py b/pip/index.py index 209c40c16..65b6ea9c8 100644 --- a/pip/index.py +++ b/pip/index.py @@ -18,7 +18,7 @@ except ImportError: import dummy_threading as threading from pip.log import logger -from pip.util import Inf, normalize_name, splitext +from pip.util import Inf, normalize_name, splitext, is_prerelease from pip.exceptions import DistributionNotFound, BestVersionAlreadyInstalled from pip.backwardcompat import (WindowsError, BytesIO, Queue, urlparse, @@ -183,6 +183,9 @@ class PackageFinder(object): logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join([''.join(s) for s in req.req.specs]))) continue + elif is_prerelease(version) and not req.prereleases: + logger.info("Ignoring link %s, version %s is a pre-release (use --pre to allow)." % (link, version)) + continue applicable_versions.append((parsed_version, link, version)) #bring the latest version to the front, but maintains the priority ordering as secondary applicable_versions = sorted(applicable_versions, key=lambda v: v[0], reverse=True) diff --git a/pip/req.py b/pip/req.py index 14aa3a0ae..7af2ad1e0 100644 --- a/pip/req.py +++ b/pip/req.py @@ -19,7 +19,7 @@ from pip.util import (display_path, rmtree, ask, ask_path_exists, backup_dir, is_installable_dir, is_local, dist_is_local, dist_in_usersite, dist_in_site_packages, renames, normalize_path, egg_link_path, make_path_relative, - call_subprocess) + call_subprocess, is_prerelease) from pip.backwardcompat import (urlparse, urllib, uses_pycache, ConfigParser, string_types, HTTPError, get_python_version, b) @@ -37,7 +37,7 @@ PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' class InstallRequirement(object): def __init__(self, req, comes_from, source_dir=None, editable=False, - url=None, as_egg=False, update=True): + url=None, as_egg=False, update=True, prereleases=None): self.extras = () if isinstance(req, string_types): req = pkg_resources.Requirement.parse(req) @@ -65,6 +65,14 @@ class InstallRequirement(object): self.uninstalled = None self.use_user_site = False + # True if pre-releases are acceptable + if prereleases: + self.prereleases = True + elif self.req is not None: + self.prereleases = any([is_prerelease(x[1]) and x[0] != "!=" for x in self.req.specs]) + else: + self.prereleases = False + @classmethod def from_editable(cls, editable_req, comes_from=None, default_vcs=None): name, url, extras_override = parse_editable(editable_req, default_vcs) @@ -73,7 +81,7 @@ class InstallRequirement(object): else: source_dir = None - res = cls(name, comes_from, source_dir=source_dir, editable=True, url=url) + res = cls(name, comes_from, source_dir=source_dir, editable=True, url=url, prereleases=True) if extras_override is not None: res.extras = extras_override @@ -81,7 +89,7 @@ class InstallRequirement(object): return res @classmethod - def from_line(cls, name, comes_from=None): + def from_line(cls, name, comes_from=None, prereleases=None): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ @@ -115,7 +123,7 @@ class InstallRequirement(object): else: req = name - return cls(req, comes_from, url=url) + return cls(req, comes_from, url=url, prereleases=prereleases) def __str__(self): if self.req: @@ -1353,7 +1361,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None): req = InstallRequirement.from_editable( line, comes_from=comes_from, default_vcs=options.default_vcs) else: - req = InstallRequirement.from_line(line, comes_from) + req = InstallRequirement.from_line(line, comes_from, prereleases=getattr(options, "pre", None)) yield req diff --git a/pip/util.py b/pip/util.py index ee2222e77..295d32729 100644 --- a/pip/util.py +++ b/pip/util.py @@ -14,6 +14,7 @@ from pip.backwardcompat import(WindowsError, string_types, raw_input, console_to_str, user_site, ssl) from pip.locations import site_packages, running_under_virtualenv, virtualenv_no_global from pip.log import logger +from pip import distlib __all__ = ['rmtree', 'display_path', 'backup_dir', 'find_command', 'ask', 'Inf', @@ -666,3 +667,20 @@ def call_subprocess(cmd, show_stdout=True, % (command_desc, proc.returncode, cwd)) if stdout is not None: return ''.join(all_output) + + +def is_prerelease(version): + """ + Attempt to determine if this is a pre-release using PEP386/PEP426 rules. + + Will return True if it is a pre-release, False is not, and None if we cannot + determine. + """ + normalized = distlib.version.suggest_normalized_version(version) + + if normalized is None: + # Cannot normalize + return + + parsed = distlib.version.normalized_key(normalized) + return any([any([y in set(["a", "b", "c", "rc", "dev"]) for y in x]) for x in parsed]) diff --git a/setup.py b/setup.py index 44b52d2b3..11119dd43 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup(name="pip", author_email='python-virtualenv@groups.google.com', url='http://www.pip-installer.org', license='MIT', - packages=['pip', 'pip.commands', 'pip.vcs', 'pip.backwardcompat'], + packages=['pip', 'pip.commands', 'pip.vcs', 'pip.backwardcompat', 'pip.distlib'], package_data={'pip': ['*.pem']}, entry_points=dict(console_scripts=['pip=pip:main', 'pip-%s=pip:main' % sys.version[:3]]), test_suite='nose.collector',