Implement a pip self-check with to determine when it's out of date.

Initial work cribbed from PR 1214.

CHANGES.txt

@@ -26,6 +26,9 @@
 
 * Added site-wide configuation files. (:pull:`1978`)
 
+* Added self-check (automatic and manual) to try to keep pip installations
+  updated (:pull:`1973`) based on @dstufft's work in :pull:`1214`.
+
 * `wsgiref` and `argparse` (for >py26) are now excluded from `pip list` and `pip
   freeze` (:pull:`1606`, :pull:`1369`)
 

pip/basecommand.py

@@ -21,7 +21,7 @@ from pip.status_codes import (
     SUCCESS, ERROR, UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND,
     PREVIOUS_BUILD_DIR_ERROR,
 )
-from pip.utils import appdirs, get_prog, normalize_path
+from pip.utils import appdirs, get_prog, normalize_path, self_check
 from pip.utils.deprecation import RemovedInPip8Warning
 from pip.utils.logging import IndentingFormatter
 
@@ -199,6 +199,11 @@ class Command(object):
                 )
                 sys.exit(VIRTUALENV_NOT_FOUND)
 
+        # Check if we're using the latest version of pip available
+        if not options.disable_self_check:
+            with self._build_session(options) as session:
+                self_check(session)
+
         try:
             status = self.run(options, args)
             # FIXME: all commands should return an exit status

pip/cmdoptions.py

@@ -390,6 +390,13 @@ no_clean = OptionMaker(
     default=False,
     help="Don't clean up build directories.")
 
+disable_self_check = OptionMaker(
+    "--disable-self-check",
+    dest="disable_self_check",
+    action="store_true",
+    default=False,
+    help="Don't periodically check PyPI to determine whether a new version "
+         "of pip is available for download.")
 
 ##########
 # groups #
@@ -417,6 +424,7 @@ general_group = {
         no_check_certificate,
         cache_dir,
         no_cache,
+        disable_self_check,
     ]
 }
 

pip/utils/__init__.py

@@ -1,6 +1,8 @@
 from __future__ import absolute_import
 
 import contextlib
+import datetime
+import json
 import locale
 import logging
 import re
@@ -17,9 +19,9 @@ from pip.exceptions import InstallationError, BadCommand
 from pip.compat import console_to_str, stdlib_pkgs
 from pip.locations import (
     site_packages, user_site, running_under_virtualenv, virtualenv_no_global,
-    write_delete_marker_file
+    write_delete_marker_file, USER_CACHE_DIR
 )
-from pip._vendor import pkg_resources, six
+from pip._vendor import pkg_resources, six, lockfile
 from pip._vendor.distlib import version
 from pip._vendor.six.moves import input
 from pip._vendor.six.moves import cStringIO
@@ -853,3 +855,121 @@ def captured_stdout():
     Taken from Lib/support/__init__.py in the CPython repo.
     """
     return captured_output('stdout')
+
+
+SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ"
+
+
+class VirtualenvSelfCheckState(object):
+    def __init__(self):
+        self.statefile_path = os.path.join(sys.prefix, "pip-selfcheck.json")
+
+        # Load the existing state
+        try:
+            with open(self.statefile_path) as statefile:
+                self.state = json.load(statefile)
+        except (IOError, ValueError):
+            self.state = {}
+
+    def save(self, pypi_version, current_time):
+        # Attempt to write out our version check file
+        with open(self.statefile_path, "w") as statefile:
+            json.dump(
+                {
+                    "last_check": current_time.strftime(SELFCHECK_DATE_FMT),
+                    "pypi_version": pypi_version,
+                },
+                statefile,
+                sort_keys=True,
+                separators=(",", ":")
+            )
+
+
+class GlobalSelfCheckState(object):
+    def __init__(self):
+        self.statefile_path = os.path.join(USER_CACHE_DIR, "selfcheck.json")
+
+        # Load the existing state
+        try:
+            with open(self.statefile_path) as statefile:
+                self.state = json.load(statefile)[sys.prefix]
+        except (IOError, ValueError, KeyError):
+            self.state = {}
+
+    def save(self, pypi_version, current_time):
+        # Attempt to write out our version check file
+        with lockfile.LockFile(self.statefile_path):
+            with open(self.statefile_path) as statefile:
+                state = json.load(statefile)
+
+            state[sys.prefix] = {
+                "last_check": current_time.strftime(SELFCHECK_DATE_FMT),
+                "pypi_version": pypi_version,
+            }
+
+            with open(self.statefile_path, "w") as statefile:
+                json.dump(state, statefile, sort_keys=True,
+                          separators=(",", ":"))
+
+
+def load_selfcheck_statefile():
+    if running_under_virtualenv():
+        return VirtualenvSelfCheckState()
+    else:
+        return GlobalSelfCheckState()
+
+
+def self_check(session):
+    """Check for an update for pip.
+
+    Limit the frequency of checks to once per week. State is stored either in
+    the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix
+    of the pip script path.
+    """
+    import pip  # imported here to prevent circular imports
+    pypi_version = None
+
+    try:
+        state = load_selfcheck_statefile()
+
+        current_time = datetime.datetime.utcnow()
+        print state.state
+        # Determine if we need to refresh the state
+        if "last_check" in state.state and "pypi_version" in state.state:
+            last_check = datetime.datetime.strptime(
+                state.state["last_check"],
+                SELFCHECK_DATE_FMT
+            )
+            print (current_time - last_check, state.state["pypi_version"])
+            if (current_time - last_check).total_seconds() < 7 * 24 * 60 * 60:
+                pypi_version = state.state["pypi_version"]
+                print 'using', pypi_version
+
+        # Refresh the version if we need to or just see if we need to warn
+        if pypi_version is None:
+            resp = session.get(
+                "https://pypi.python.org/pypi/pip/json",
+                headers={"Accept": "application/json"},
+            )
+            resp.raise_for_status()
+            pypi_version = resp.json()["info"]["version"]
+
+            # save that we've performed a check
+            state.save(pypi_version, current_time)
+
+        pip_version = pkg_resources.parse_version(pip.__version__)
+
+        # Determine if our pypi_version is older
+        if pip_version < pkg_resources.parse_version(pypi_version):
+            logger.warn(
+                "You are using pip version %s, however version %s is "
+                "available.\nYou should consider upgrading via the "
+                "'pip install --upgrade pip' command." % (pip.__version__,
+                                                          pypi_version)
+            )
+
+    except Exception:
+        logger.debug(
+            "There was an error checking the latest version of pip",
+            exc_info=True,
+        )

tests/unit/test_basecommand.py

@@ -11,6 +11,10 @@ class FakeCommand(Command):
         self.error = error
         super(FakeCommand, self).__init__()
 
+    def main(self, args):
+        args.append("--disable-self-check")
+        return super(FakeCommand, self).main(args)
+
     def run(self, options, args):
         logging.getLogger("pip.tests").info("fake")
         if self.error:

tests/unit/test_self_check.py

@@ -0,0 +1,150 @@
+import sys
+import datetime
+import os
+from contextlib import contextmanager
+
+import pytest
+import pretend
+
+import pip
+from pip._vendor import lockfile
+from pip import util
+
+
+@pytest.mark.parametrize(
+    ['stored_time', 'newver', 'check', 'warn'],
+    [
+        ('1970-01-01T10:00:00Z', '2.0', True, True),
+        ('1970-01-01T10:00:00Z', '1.0', True, False),
+        ('1970-01-06T10:00:00Z', '1.0', False, False),
+        ('1970-01-06T10:00:00Z', '2.0', False, True),
+    ]
+)
+def test_self_check(monkeypatch, stored_time, newver, check, warn):
+    monkeypatch.setattr(pip, '__version__', '1.0')
+
+    resp = pretend.stub(
+        raise_for_status=pretend.call_recorder(lambda: None),
+        json=pretend.call_recorder(lambda: {"info": {"version": newver}}),
+    )
+    session = pretend.stub(
+        get=pretend.call_recorder(lambda u, headers=None: resp),
+    )
+
+    fake_state = pretend.stub(
+        state={"last_check": stored_time, 'pypi_version': '1.0'},
+        save=pretend.call_recorder(lambda v, t: None),
+    )
+
+    monkeypatch.setattr(util, 'load_selfcheck_statefile', lambda: fake_state)
+
+    fake_now = datetime.datetime(1970, 1, 9, 10, 00, 00)
+
+    fake_datetime = pretend.stub(
+        utcnow=pretend.call_recorder(lambda: fake_now),
+        strptime=datetime.datetime.strptime,
+    )
+    monkeypatch.setattr(datetime, 'datetime', fake_datetime)
+
+    monkeypatch.setattr(util.logger, 'warn',
+                        pretend.call_recorder(lambda s: None))
+    monkeypatch.setattr(util.logger, 'debug',
+                        pretend.call_recorder(lambda s, exc_info=None: None))
+
+    util.self_check(session)
+
+    assert not util.logger.debug.calls
+
+    if check:
+        assert session.get.calls == [pretend.call(
+            "https://pypi.python.org/pypi/pip/json",
+            headers={"Accept": "application/json"}
+        )]
+        assert fake_state.save.calls == [pretend.call(newver, fake_now)]
+        if warn:
+            assert len(util.logger.warn.calls) == 1
+        else:
+            assert len(util.logger.warn.calls) == 0
+    else:
+        assert session.get.calls == []
+        assert fake_state.save.calls == []
+
+
+def test_virtualenv_state(monkeypatch):
+    CONTENT = '{"last_check": "1970-01-02T11:00:00Z", "pypi_version": "1.0"}'
+    fake_file = pretend.stub(
+        read=pretend.call_recorder(lambda: CONTENT),
+        write=pretend.call_recorder(lambda s: None),
+    )
+
+    @pretend.call_recorder
+    @contextmanager
+    def fake_open(filename, mode='r'):
+        yield fake_file
+
+    monkeypatch.setattr(util, 'open', fake_open, raising=False)
+
+    monkeypatch.setattr(util, 'running_under_virtualenv',
+                        pretend.call_recorder(lambda: True))
+
+    monkeypatch.setattr(sys, 'prefix', 'virtually_env')
+
+    state = util.load_selfcheck_statefile()
+    state.save('2.0', datetime.datetime.utcnow())
+
+    assert len(util.running_under_virtualenv.calls) == 1
+
+    expected_path = os.path.join('virtually_env', 'pip-selfcheck.json')
+    assert fake_open.calls == [
+        pretend.call(expected_path),
+        pretend.call(expected_path, 'w'),
+    ]
+
+    # json.dumps will call this a number of times
+    assert len(fake_file.write.calls)
+
+
+def test_global_state(monkeypatch):
+    CONTENT = '''{"pip_prefix": {"last_check": "1970-01-02T11:00:00Z",
+        "pypi_version": "1.0"}}'''
+    fake_file = pretend.stub(
+        read=pretend.call_recorder(lambda: CONTENT),
+        write=pretend.call_recorder(lambda s: None),
+    )
+
+    @pretend.call_recorder
+    @contextmanager
+    def fake_open(filename, mode='r'):
+        yield fake_file
+
+    monkeypatch.setattr(util, 'open', fake_open, raising=False)
+
+    @pretend.call_recorder
+    @contextmanager
+    def fake_lock(filename):
+        yield
+
+    monkeypatch.setattr(lockfile, 'LockFile', fake_lock)
+
+    monkeypatch.setattr(util, 'running_under_virtualenv',
+                        pretend.call_recorder(lambda: False))
+
+    monkeypatch.setattr(util, 'USER_CACHE_DIR', 'cache_dir')
+    monkeypatch.setattr(sys, 'prefix', 'pip_prefix')
+
+    state = util.load_selfcheck_statefile()
+    state.save('2.0', datetime.datetime.utcnow())
+
+    assert len(util.running_under_virtualenv.calls) == 1
+
+    expected_path = os.path.join('cache_dir', 'selfcheck.json')
+    assert fake_lock.calls == [pretend.call(expected_path)]
+
+    assert fake_open.calls == [
+        pretend.call(expected_path),
+        pretend.call(expected_path),
+        pretend.call(expected_path, 'w'),
+    ]
+
+    # json.dumps will call this a number of times
+    assert len(fake_file.write.calls)