From 52ca02608e339e96b6170ef5a0dcdcefc9a327e2 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 24 Apr 2014 07:29:57 -0400 Subject: [PATCH] Use CacheControl instead of custom cache code * Deprecates the --download-cache option & removes the download cache code. * Removes the in memory page cache on the index * Uses CacheControl to cache all cacheable HTTP requests to the filesystem. * Properly handles CacheControl headers for unconditional caching. * Will use ETag and Last-Modified headers to attempt to do a conditional HTTP request to speed up cache misses and turn them into cache hits. * Removes some concurrency unsafe code in the download cache accesses. * Uses a Cache-Control request header to limit the maximum length of time a cache is valid for. * Adds pip.appdirs to handle platform specific application directories such as cache, config, data, etc. --- CHANGES.txt | 8 ++ docs/reference/pip_install.rst | 50 +++++++----- pip/appdirs.py | 138 +++++++++++++++++++++++++++++++++ pip/basecommand.py | 7 +- pip/cmdoptions.py | 24 +++++- pip/commands/install.py | 9 ++- pip/commands/wheel.py | 9 ++- pip/download.py | 110 +++++++++++++------------- pip/index.py | 82 ++++---------------- pip/locations.py | 4 + pip/req/req_set.py | 14 ++-- pip/util.py | 20 +---- tests/unit/test_appdirs.py | 45 +++++++++++ tests/unit/test_download.py | 114 +++++++++++++++------------ tests/unit/test_req.py | 1 - tox.ini | 1 + 16 files changed, 412 insertions(+), 224 deletions(-) create mode 100644 pip/appdirs.py create mode 100644 tests/unit/test_appdirs.py diff --git a/CHANGES.txt b/CHANGES.txt index 96b408992..2fe6c9785 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -37,6 +37,14 @@ * Fixed :issue:`1775`. `pip wheel` wasn't building wheels for dependencies of editable requirements. +* **DEPRECATION** ``pip install --download-cache`` and + ``pip wheel --download-cache`` command line flags have been deprecated and + the functionality removed. Since pip now automatically configures and uses + it's internal HTTP cache which supplants the ``--download-cache`` the + existing options have been made non functional but will still be accepted + until their removal in pip v1.8. For more information please see + https://pip.pypa.io/en/latest/reference/pip_install.html#caching + **1.5.5 (2014-05-03)** diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 2e672ea68..743704461 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -249,6 +249,36 @@ Starting with v1.3, pip provides SSL certificate verification over https, for th of providing secure, certified downloads from PyPI. +.. _`Caching`: + +Caching ++++++++ + +Starting with v1.6, pip provides an on by default cache which functions +similarly to that of a web browser. While the cache is on by default and is +designed do the right thing by default you can disable the cache and always +access PyPI by utilizing the ``--no-cache-dir`` option. + +When making any HTTP request pip will first check it's local cache to determine +if it has a suitable response stored for that request which has not expired. If +it does then it simply returns that response and doesn't make the request. + +If it has a response stored, but it has expired, then it will attempt to make a +conditional request to refresh the cache which will either return an empty +response telling pip to simply use the cached item (and refresh the expiration +timer) or it will return a whole new response which pip can then store in the +cache. + +When storing items in the cache pip will respect the ``CacheControl`` header +if it exists, or it will fall back to the ``Expires`` header if that exists. +This allows pip to function as a browser would, and allows the index server +to communicate to pip how long it is reasonable to cache any particular item. + +While this cache attempts to minimize network activity, it does not prevent +network access all together. If you want a fast/local install solution that +circumvents accessing PyPI, see :ref:`Fast & Local Installs`. + + Hash Verification +++++++++++++++++ @@ -264,26 +294,6 @@ It is not intended to provide security against tampering. For that, see :ref:`SSL Certificate Verification` -Download Cache -++++++++++++++ - -pip offers a :ref:`--download-cache ` option for -installs to prevent redundant downloads of archives from PyPI. - -The point of this cache is *not* to circumvent the index crawling process, but -to *just* prevent redundant downloads. - -Items are stored in this cache based on the url the archive was found at, not -simply the archive name. - -If you want a fast/local install solution that circumvents crawling PyPI, see -the :ref:`Fast & Local Installs`. - -Like all options, :ref:`--download-cache `, can also -be set as an environment variable, or placed into the pip config file. See the -:ref:`Configuration` section. - - .. _`editable-installs`: "Editable" Installs diff --git a/pip/appdirs.py b/pip/appdirs.py new file mode 100644 index 000000000..790ab8c88 --- /dev/null +++ b/pip/appdirs.py @@ -0,0 +1,138 @@ +""" +This code was taken from https://github.com/ActiveState/appdirs and modified +to suite our purposes. +""" +import os +import sys + +from pip._vendor import six + + +def user_cache_dir(appname): + r""" + Return full path to the user-specific cache dir for this application. + + "appname" is the name of application. + + Typical user cache directories are: + Mac OS X: ~/Library/Caches/ + Unix: ~/.cache/ (XDG default) + Windows: C:\Users\\AppData\Local\\Cache + + On Windows the only suggestion in the MSDN docs is that local settings go + in the `CSIDL_LOCAL_APPDATA` directory. This is identical to the + non-roaming app data dir (the default returned by `user_data_dir`). Apps + typically put cache data somewhere *under* the given dir here. Some + examples: + ...\Mozilla\Firefox\Profiles\\Cache + ...\Acme\SuperApp\Cache\1.0 + + OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. + """ + if sys.platform == "win32": + # Get the base path + path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) + + # Add our app name and Cache directory to it + path = os.path.join(path, appname, "Cache") + elif sys.platform == "darwin": + # Get the base path + path = os.path.expanduser("~/Library/Caches") + + # Add our app name to it + path = os.path.join(path, appname) + else: + # Get the base path + path = os.getenv("XDG_CACHE_HOME", os.path.expanduser("~/.cache")) + + # Add our app name to it + path = os.path.join(path, appname) + + return path + + +# -- Windows support functions -- + +def _get_win_folder_from_registry(csidl_name): + """ + This is a fallback technique at best. I'm not sure if using the + registry for this guarantees us the correct answer for all CSIDL_* + names. + """ + import _winreg + + shell_folder_name = { + "CSIDL_APPDATA": "AppData", + "CSIDL_COMMON_APPDATA": "Common AppData", + "CSIDL_LOCAL_APPDATA": "Local AppData", + }[csidl_name] + + key = _winreg.OpenKey( + _winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" + ) + directory, _type = _winreg.QueryValueEx(key, shell_folder_name) + return directory + + +def _get_win_folder_with_pywin32(csidl_name): + from win32com.shell import shellcon, shell + directory = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) + # Try to make this a unicode path because SHGetFolderPath does + # not return unicode strings when there is unicode data in the + # path. + try: + directory = six.text_type(directory) + + # Downgrade to short path name if have highbit chars. See + # . + has_high_char = False + for c in directory: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + try: + import win32api + directory = win32api.GetShortPathName(directory) + except ImportError: + pass + except UnicodeError: + pass + return directory + + +def _get_win_folder_with_ctypes(csidl_name): + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + }[csidl_name] + + buf = ctypes.create_unicode_buffer(1024) + ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if have highbit chars. See + # . + has_high_char = False + for c in buf: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf2 = ctypes.create_unicode_buffer(1024) + if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + return buf.value + +if sys.platform == "win32": + try: + import win32com.shell # noqa + _get_win_folder = _get_win_folder_with_pywin32 + except ImportError: + try: + import ctypes + _get_win_folder = _get_win_folder_with_ctypes + except ImportError: + _get_win_folder = _get_win_folder_from_registry diff --git a/pip/basecommand.py b/pip/basecommand.py index 22df0156f..2b611215d 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -19,7 +19,7 @@ from pip.status_codes import ( SUCCESS, ERROR, UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND, PREVIOUS_BUILD_DIR_ERROR, ) -from pip.util import get_prog +from pip.util import get_prog, normalize_path __all__ = ['Command'] @@ -54,7 +54,10 @@ class Command(object): self.parser.add_option_group(gen_opts) def _build_session(self, options): - session = PipSession(retries=options.retries) + session = PipSession( + cache=normalize_path(os.path.join(options.cache_dir, "http")), + retries=options.retries, + ) # Handle custom ca-bundles from the user if options.cert: diff --git a/pip/cmdoptions.py b/pip/cmdoptions.py index 08b6069f4..3982dc2a8 100644 --- a/pip/cmdoptions.py +++ b/pip/cmdoptions.py @@ -9,7 +9,9 @@ pass on state. To be consistent, all options will follow this design. """ import copy from optparse import OptionGroup, SUPPRESS_HELP, Option -from pip.locations import build_prefix, default_log_file, src_prefix +from pip.locations import ( + USER_CACHE_DIR, build_prefix, default_log_file, src_prefix, +) def make_option_group(group, parser): @@ -322,12 +324,26 @@ no_use_wheel = OptionMaker( 'find-links locations.'), ) +cache_dir = OptionMaker( + "--cache-dir", + dest="cache_dir", + default=USER_CACHE_DIR, + metavar="dir", + help="Store the cache data in ." +) + +no_cache = OptionMaker( + "--no-cache-dir", + dest="cache_dir", + action="store_false", + help="Disable the cache.", +) + download_cache = OptionMaker( '--download-cache', dest='download_cache', - metavar='dir', default=None, - help='Cache downloaded packages in .') + help=SUPPRESS_HELP) no_deps = OptionMaker( '--no-deps', '--no-dependencies', @@ -396,6 +412,8 @@ general_group = { cert, client_cert, no_check_certificate, + cache_dir, + no_cache, ] } diff --git a/pip/commands/install.py b/pip/commands/install.py index 3fcc279c0..20b54b9e5 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -248,6 +248,14 @@ class InstallCommand(Command): ) index_urls += options.mirrors + if options.download_cache: + logger.deprecated( + "1.8", + "--download-cache has been deprecated and will be removed in " + " the future. Pip now automatically uses and configures its " + "cache." + ) + session = self._build_session(options) finder = self._build_package_finder(options, index_urls, session) @@ -256,7 +264,6 @@ class InstallCommand(Command): build_dir=options.build_dir, src_dir=options.src_dir, download_dir=options.download_dir, - download_cache=options.download_cache, upgrade=options.upgrade, as_egg=options.as_egg, ignore_installed=options.ignore_installed, diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index c86d2af21..ebc724f8a 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -144,6 +144,14 @@ class WheelCommand(Command): ) index_urls += options.mirrors + if options.download_cache: + logger.deprecated( + "1.8", + "--download-cache has been deprecated and will be removed in " + " the future. Pip now automatically uses and configures its " + "cache." + ) + session = self._build_session(options) finder = PackageFinder( @@ -162,7 +170,6 @@ class WheelCommand(Command): build_dir=options.build_dir, src_dir=options.src_dir, download_dir=None, - download_cache=options.download_cache, ignore_dependencies=options.ignore_dependencies, ignore_installed=True, session=session, diff --git a/pip/download.py b/pip/download.py index 53d57f1e8..c517ce37c 100644 --- a/pip/download.py +++ b/pip/download.py @@ -15,17 +15,20 @@ import pip from pip.backwardcompat import urllib, urlparse, raw_input from pip.exceptions import InstallationError, HashMismatch from pip.util import (splitext, rmtree, format_size, display_path, - backup_dir, ask_path_exists, unpack_file, - create_download_cache_folder, cache_download) + backup_dir, ask_path_exists, unpack_file) from pip.vcs import vcs from pip.log import logger from pip._vendor import requests, six -from pip._vendor.requests.adapters import BaseAdapter +from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth from pip._vendor.requests.compat import IncompleteRead from pip._vendor.requests.exceptions import ChunkedEncodingError from pip._vendor.requests.models import Response from pip._vendor.requests.structures import CaseInsensitiveDict +from pip._vendor.cachecontrol import CacheControlAdapter +from pip._vendor.cachecontrol.caches import FileCache +from pip._vendor.lockfile import LockError + __all__ = ['get_file_content', 'is_url', 'url_to_path', 'path_to_url', @@ -177,12 +180,47 @@ class LocalFSAdapter(BaseAdapter): pass +class SafeFileCache(FileCache): + """ + A file based cache which is safe to use even when the target directory may + not be accessible or writable. + """ + + def get(self, *args, **kwargs): + try: + return super(SafeFileCache, self).get(*args, **kwargs) + except (LockError, OSError, IOError): + # We intentionally silence this error, if we can't access the cache + # then we can just skip caching and process the request as if + # caching wasn't enabled. + pass + + def set(self, *args, **kwargs): + try: + return super(SafeFileCache, self).set(*args, **kwargs) + except (LockError, OSError, IOError): + # We intentionally silence this error, if we can't access the cache + # then we can just skip caching and process the request as if + # caching wasn't enabled. + pass + + def delete(self, *args, **kwargs): + try: + return super(SafeFileCache, self).delete(*args, **kwargs) + except (LockError, OSError, IOError): + # We intentionally silence this error, if we can't access the cache + # then we can just skip caching and process the request as if + # caching wasn't enabled. + pass + + class PipSession(requests.Session): timeout = None def __init__(self, *args, **kwargs): - retries = kwargs.pop('retries', None) + retries = kwargs.pop("retries", 0) + cache = kwargs.pop("cache", None) super(PipSession, self).__init__(*args, **kwargs) @@ -192,11 +230,16 @@ class PipSession(requests.Session): # Attach our Authentication handler to the session self.auth = MultiDomainBasicAuth() - # Configure retries - if retries: - http_adapter = requests.adapters.HTTPAdapter(max_retries=retries) - self.mount("http://", http_adapter) - self.mount("https://", http_adapter) + if cache: + http_adapter = CacheControlAdapter( + cache=SafeFileCache(cache), + max_retries=retries, + ) + else: + http_adapter = HTTPAdapter(max_retries=retries) + + self.mount("http://", http_adapter) + self.mount("https://", http_adapter) # Enable file:// urls self.mount("file://", LocalFSAdapter()) @@ -383,6 +426,7 @@ def _download_url(resp, link, temp_location): "Unsupported hash name %s for package %s" % (link.hash_name, link) ) + try: total_length = int(resp.headers['content-length']) except (ValueError, KeyError, TypeError): @@ -493,8 +537,7 @@ def _copy_file(filename, location, content_type, link): logger.notify('Saved %s' % display_path(download_location)) -def unpack_http_url(link, location, download_cache, download_dir=None, - session=None): +def unpack_http_url(link, location, download_dir=None, session=None): if session is None: raise TypeError( "unpack_http_url() missing 1 required keyword argument: 'session'" @@ -503,24 +546,8 @@ def unpack_http_url(link, location, download_cache, download_dir=None, temp_dir = tempfile.mkdtemp('-unpack', 'pip-') temp_location = None target_url = link.url.split('#', 1)[0] - already_cached = False - cache_file = None - cache_content_type_file = None - download_hash = None - # If a download cache is specified, is the file cached there? - if download_cache: - cache_file = os.path.join( - download_cache, - urllib.quote(target_url, '') - ) - cache_content_type_file = cache_file + '.content-type' - already_cached = ( - os.path.exists(cache_file) and - os.path.exists(cache_content_type_file) - ) - if not os.path.isdir(download_cache): - create_download_cache_folder(download_cache) + download_hash = None # If a download dir is specified, is the file already downloaded there? already_downloaded = None @@ -547,27 +574,6 @@ def unpack_http_url(link, location, download_cache, download_dir=None, os.unlink(already_downloaded) already_downloaded = None - # If not a valid download, let's confirm the cached file is valid - if already_cached and not temp_location: - with open(cache_content_type_file) as fp: - content_type = fp.read().strip() - temp_location = cache_file - logger.notify('Using download cache from %s' % cache_file) - if link.hash and link.hash_name: - download_hash = _get_hash_from_file(cache_file, link) - try: - _check_hash(download_hash, link) - except HashMismatch: - logger.warn( - 'Cached file %s has bad hash, ' - 're-downloading.' % temp_location - ) - temp_location = None - os.unlink(cache_file) - os.unlink(cache_content_type_file) - already_cached = False - - # We don't have either a cached or a downloaded copy # let's download to a tmp dir if not temp_location: try: @@ -632,11 +638,7 @@ def unpack_http_url(link, location, download_cache, download_dir=None, # archives, they have to be unpacked to parse dependencies unpack_file(temp_location, location, content_type, link) - # if using a download cache, cache it, if needed - if cache_file and not already_cached: - cache_download(cache_file, temp_location, content_type) - - if not (already_cached or already_downloaded): + if not already_downloaded: os.unlink(temp_location) os.rmdir(temp_dir) diff --git a/pip/index.py b/pip/index.py index 5a3044f5e..b9e1f24ac 100644 --- a/pip/index.py +++ b/pip/index.py @@ -47,7 +47,7 @@ class PackageFinder(object): self.find_links = find_links self.index_urls = index_urls - self.cache = PageCache() + # These are boring links that have already been logged somehow: self.logged_links = set() @@ -187,8 +187,7 @@ class PackageFinder(object): mkurl_pypi_url(self.index_urls[0]), trusted=True, ) - # This will also cache the page, so it's okay that we get it again - # later: + page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name( @@ -662,41 +661,7 @@ class PackageFinder(object): return None def _get_page(self, link, req): - return HTMLPage.get_page( - link, req, - cache=self.cache, - session=self.session, - ) - - -class PageCache(object): - """Cache of HTML pages""" - - failure_limit = 3 - - def __init__(self): - self._failures = {} - self._pages = {} - self._archives = {} - - def too_many_failures(self, url): - return self._failures.get(url, 0) >= self.failure_limit - - def get_page(self, url): - return self._pages.get(url) - - def is_archive(self, url): - return self._archives.get(url, False) - - def set_is_archive(self, url, value=True): - self._archives[url] = value - - def add_page_failure(self, url, level): - self._failures[url] = self._failures.get(url, 0) + level - - def add_page(self, urls, page): - for url in urls: - self._pages[url] = page + return HTMLPage.get_page(link, req, session=self.session) class HTMLPage(object): @@ -721,7 +686,7 @@ class HTMLPage(object): return self.url @classmethod - def get_page(cls, link, req, cache=None, skip_archives=True, session=None): + def get_page(cls, link, req, skip_archives=True, session=None): if session is None: raise TypeError( "get_page() missing 1 required keyword argument: 'session'" @@ -729,8 +694,6 @@ class HTMLPage(object): url = link.url url = url.split('#', 1)[0] - if cache.too_many_failures(url): - return None # Check for VCS schemes that do not support lookup as web pages. from pip.vcs import VcsSupport @@ -741,15 +704,8 @@ class HTMLPage(object): ) return None - if cache is not None: - inst = cache.get_page(url) - if inst is not None: - return inst try: if skip_archives: - if cache is not None: - if cache.is_archive(url): - return None filename = link.filename for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']: if filename.endswith(bad_ext): @@ -763,9 +719,8 @@ class HTMLPage(object): 'Skipping page %s because of Content-Type: ' '%s' % (link, content_type) ) - if cache is not None: - cache.set_is_archive(url) - return None + return + logger.debug('Getting page %s' % url) # Tack index.html onto file:// URLs that point to directories @@ -779,7 +734,13 @@ class HTMLPage(object): url = urlparse.urljoin(url, 'index.html') logger.debug(' file: URL is directory, getting %s' % url) - resp = session.get(url, headers={"Accept": "text/html"}) + resp = session.get( + url, + headers={ + "Accept": "text/html", + "Cache-Control": "max-age=600", + }, + ) resp.raise_for_status() # The check for archives above only works if the url ends with @@ -793,37 +754,31 @@ class HTMLPage(object): 'Skipping page %s because of Content-Type: %s' % (link, content_type) ) - if cache is not None: - cache.set_is_archive(url) - return None + return inst = cls(resp.text, resp.url, resp.headers, trusted=link.trusted) except requests.HTTPError as exc: level = 2 if exc.response.status_code == 404 else 1 - cls._handle_fail(req, link, exc, url, cache=cache, level=level) + cls._handle_fail(req, link, exc, url, level=level) except requests.ConnectionError as exc: cls._handle_fail( req, link, "connection error: %s" % exc, url, - cache=cache, ) except requests.Timeout: - cls._handle_fail(req, link, "timed out", url, cache=cache) + cls._handle_fail(req, link, "timed out", url) except SSLError as exc: reason = ("There was a problem confirming the ssl certificate: " "%s" % exc) cls._handle_fail( req, link, reason, url, - cache=cache, level=2, meth=logger.notify, ) else: - if cache is not None: - cache.add_page([url, resp.url], inst) return inst @staticmethod - def _handle_fail(req, link, reason, url, cache=None, level=1, meth=None): + def _handle_fail(req, link, reason, url, level=1, meth=None): if meth is None: meth = logger.info @@ -831,9 +786,6 @@ class HTMLPage(object): meth("Will skip URL %s when looking for download links for %s" % (link.url, req)) - if cache is not None: - cache.add_page_failure(url, level) - @staticmethod def _get_content_type(url, session): """Get the Content-Type of the given url, using a HEAD request""" diff --git a/pip/locations.py b/pip/locations.py index 7a848cf11..6e0f43e85 100644 --- a/pip/locations.py +++ b/pip/locations.py @@ -9,6 +9,7 @@ import tempfile from distutils import sysconfig from distutils.command.install import install, SCHEME_KEYS +from pip import appdirs from pip.backwardcompat import get_path_uid import pip.exceptions @@ -16,6 +17,9 @@ import pip.exceptions # Hack for flake8 install +# Application Directories +USER_CACHE_DIR = appdirs.user_cache_dir("pip") + DELETE_MARKER_MESSAGE = '''\ This file is placed here by pip to indicate the source was put diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 50c225801..23e231a44 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -48,11 +48,11 @@ class Requirements(object): class RequirementSet(object): - def __init__(self, build_dir, src_dir, download_dir, download_cache=None, - upgrade=False, ignore_installed=False, as_egg=False, - target_dir=None, ignore_dependencies=False, - force_reinstall=False, use_user_site=False, session=None, - pycompile=True, wheel_download_dir=None): + def __init__(self, build_dir, src_dir, download_dir, upgrade=False, + ignore_installed=False, as_egg=False, target_dir=None, + ignore_dependencies=False, force_reinstall=False, + use_user_site=False, session=None, pycompile=True, + wheel_download_dir=None): if session is None: raise TypeError( "RequirementSet() missing 1 required keyword argument: " @@ -62,9 +62,6 @@ class RequirementSet(object): self.build_dir = build_dir self.src_dir = src_dir self.download_dir = download_dir - if download_cache: - download_cache = normalize_path(download_cache) - self.download_cache = download_cache self.upgrade = upgrade self.ignore_installed = ignore_installed self.force_reinstall = force_reinstall @@ -527,7 +524,6 @@ class RequirementSet(object): unpack_http_url( link, location, - self.download_cache, download_dir, self.session, ) diff --git a/pip/util.py b/pip/util.py index eadcd5c5d..7a4539d01 100644 --- a/pip/util.py +++ b/pip/util.py @@ -29,8 +29,7 @@ __all__ = ['rmtree', 'display_path', 'backup_dir', 'split_leading_dir', 'has_leading_dir', 'make_path_relative', 'normalize_path', 'renames', 'get_terminal_size', 'get_prog', - 'unzip_file', 'untar_file', 'create_download_cache_folder', - 'cache_download', 'unpack_file', 'call_subprocess'] + 'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess'] def get_prog(): @@ -610,23 +609,6 @@ def untar_file(filename, location): tar.close() -def create_download_cache_folder(folder): - logger.indent -= 2 - logger.notify('Creating supposed download cache at %s' % folder) - logger.indent += 2 - os.makedirs(folder) - - -def cache_download(target_file, temp_location, content_type): - logger.notify( - 'Storing download in cache at %s' % display_path(target_file) - ) - shutil.copyfile(temp_location, target_file) - fp = open(target_file + '.content-type', 'w') - fp.write(content_type) - fp.close() - - def unpack_file(filename, location, content_type, link): filename = os.path.realpath(filename) if (content_type == 'application/zip' diff --git a/tests/unit/test_appdirs.py b/tests/unit/test_appdirs.py new file mode 100644 index 000000000..e3b996a09 --- /dev/null +++ b/tests/unit/test_appdirs.py @@ -0,0 +1,45 @@ +import sys + +import pretend + +from pip import appdirs + + +class TestUserCacheDir: + + def test_user_cache_dir_win(self, monkeypatch): + @pretend.call_recorder + def _get_win_folder(base): + return "C:\\Users\\test\\AppData\\Local" + + monkeypatch.setattr( + appdirs, + "_get_win_folder", + _get_win_folder, + raising=False, + ) + monkeypatch.setattr(sys, "platform", "win32") + + assert (appdirs.user_cache_dir("pip").replace("/", "\\") + == "C:\\Users\\test\\AppData\\Local\\pip\\Cache") + assert _get_win_folder.calls == [pretend.call("CSIDL_LOCAL_APPDATA")] + + def test_user_cache_dir_osx(self, monkeypatch): + monkeypatch.setenv("HOME", "/home/test") + monkeypatch.setattr(sys, "platform", "darwin") + + assert appdirs.user_cache_dir("pip") == "/home/test/Library/Caches/pip" + + def test_user_cache_dir_linux(self, monkeypatch): + monkeypatch.delenv("XDG_CACHE_HOME") + monkeypatch.setenv("HOME", "/home/test") + monkeypatch.setattr(sys, "platform", "linux2") + + assert appdirs.user_cache_dir("pip") == "/home/test/.cache/pip" + + def test_user_cache_dir_linux_override(self, monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", "/home/test/.other-cache") + monkeypatch.setenv("HOME", "/home/test") + monkeypatch.setattr(sys, "platform", "linux2") + + assert appdirs.user_cache_dir("pip") == "/home/test/.other-cache/pip" diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index 353a51ee9..3f3b99d65 100644 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -7,10 +7,12 @@ from mock import Mock, patch import pytest import pip -from pip.backwardcompat import urllib, BytesIO, b, pathname2url +from pip.backwardcompat import BytesIO, b, pathname2url from pip.exceptions import HashMismatch -from pip.download import (PipSession, path_to_url, unpack_http_url, - url_to_path, unpack_file_url) +from pip.download import ( + PipSession, SafeFileCache, path_to_url, unpack_http_url, url_to_path, + unpack_file_url, +) from pip.index import Link @@ -35,7 +37,6 @@ def test_unpack_http_url_with_urllib_response_without_content_type(data): unpack_http_url( link, temp_dir, - download_cache=None, download_dir=None, session=session, ) @@ -76,50 +77,6 @@ class MockResponse(object): pass -@patch('pip.download.unpack_file') -def test_unpack_http_url_bad_cache_checksum(mock_unpack_file): - """ - If cached download has bad checksum, re-download. - """ - base_url = 'http://www.example.com/somepackage.tgz' - contents = b('downloaded') - download_hash = hashlib.new('sha1', contents) - link = Link(base_url + '#sha1=' + download_hash.hexdigest()) - - session = Mock() - session.get = Mock() - response = session.get.return_value = MockResponse(contents) - response.headers = {'content-type': 'application/x-tar'} - response.url = base_url - - cache_dir = mkdtemp() - try: - cache_file = os.path.join(cache_dir, urllib.quote(base_url, '')) - cache_ct_file = cache_file + '.content-type' - _write_file(cache_file, 'some contents') - _write_file(cache_ct_file, 'application/x-tar') - - unpack_http_url( - link, - 'location', - download_cache=cache_dir, - session=session, - ) - - # despite existence of cached file with bad hash, downloaded again - session.get.assert_called_once_with( - "http://www.example.com/somepackage.tgz", - headers={"Accept-Encoding": "identity"}, - stream=True, - ) - # cached file is replaced with newly downloaded file - with open(cache_file) as fh: - assert fh.read() == 'downloaded' - - finally: - rmtree(cache_dir) - - @patch('pip.download.unpack_file') def test_unpack_http_url_bad_downloaded_checksum(mock_unpack_file): """ @@ -144,7 +101,6 @@ def test_unpack_http_url_bad_downloaded_checksum(mock_unpack_file): unpack_http_url( link, 'location', - download_cache=None, download_dir=download_dir, session=session, ) @@ -282,3 +238,63 @@ class Test_unpack_file_url(object): unpack_file_url(dist_url, self.build_dir, download_dir=self.download_dir) assert os.path.isdir(os.path.join(self.build_dir, 'fspkg')) + + +class TestSafeFileCache: + + def test_cache_roundtrip(self, tmpdir): + cache_dir = tmpdir.join("test-cache") + cache_dir.makedirs() + + cache = SafeFileCache(cache_dir) + assert cache.get("test key") is None + cache.set("test key", b"a test string") + assert cache.get("test key") == b"a test string" + cache.delete("test key") + assert cache.get("test key") is None + + def test_safe_get_no_perms(self, tmpdir, monkeypatch): + cache_dir = tmpdir.join("unreadable-cache") + cache_dir.makedirs() + os.chmod(cache_dir, 000) + + monkeypatch.setattr(os.path, "exists", lambda x: True) + + cache = SafeFileCache(cache_dir) + cache.get("foo") + + def test_safe_set_no_perms(self, tmpdir): + cache_dir = tmpdir.join("unreadable-cache") + cache_dir.makedirs() + os.chmod(cache_dir, 000) + + cache = SafeFileCache(cache_dir) + cache.set("foo", "bar") + + def test_safe_delete_no_perms(self, tmpdir): + cache_dir = tmpdir.join("unreadable-cache") + cache_dir.makedirs() + os.chmod(cache_dir, 000) + + cache = SafeFileCache(cache_dir) + cache.delete("foo") + + +class TestPipSession: + + def test_cache_defaults_off(self): + session = PipSession() + + assert not hasattr(session.adapters["http://"], "cache") + assert not hasattr(session.adapters["https://"], "cache") + + def test_cache_is_enabled(self, tmpdir): + session = PipSession(cache=tmpdir.join("test-cache")) + + assert hasattr(session.adapters["http://"], "cache") + assert hasattr(session.adapters["https://"], "cache") + + assert (session.adapters["http://"].cache.directory + == tmpdir.join("test-cache")) + assert (session.adapters["https://"].cache.directory + == tmpdir.join("test-cache")) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 1b6dd077e..a44e98bc4 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -34,7 +34,6 @@ class TestRequirementSet(object): build_dir=os.path.join(self.tempdir, 'build'), src_dir=os.path.join(self.tempdir, 'src'), download_dir=None, - download_cache=os.path.join(self.tempdir, 'download_cache'), session=PipSession(), ) diff --git a/tox.ini b/tox.ini index a194e4a9e..6f9a020bf 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ envlist = [testenv] deps = + pretend pytest pytest-xdist mock