McSinyx
/
pip
mirror of https://github.com/pypa/pip
1
1
Fork 0
Code Issues Releases Activity

Move SafeFileCache to network.cache.

This commit is contained in:
Chris Hunt 2019-09-21 16:04:46 -04:00
parent 43864b86bb
commit 99c29af3ce
4 changed files with 135 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
src/pip/_internal/download.py
View File

@ -10,12 +10,9 @@ import platform
import re
import shutil
import sys
from contextlib import contextmanager
from pip._vendor import requests, six, urllib3
from pip._vendor.cachecontrol import CacheControlAdapter
from pip._vendor.cachecontrol.cache import BaseCache
from pip._vendor.cachecontrol.caches import FileCache
from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
from pip._vendor.requests.structures import CaseInsensitiveDict
@ -29,15 +26,11 @@ import pip
from pip._internal.exceptions import HashMismatch, InstallationError
from pip._internal.models.index import PyPI
from pip._internal.network.auth import MultiDomainBasicAuth
from pip._internal.network.cache import SafeFileCache
# Import ssl from compat so the initial import occurs in only one place.
from pip._internal.utils.compat import HAS_TLS, ipaddress, ssl
from pip._internal.utils.encoding import auto_decode
from pip._internal.utils.filesystem import (
adjacent_tmp_file,
check_path_owner,
copy2_fixed,
replace,
)
from pip._internal.utils.filesystem import check_path_owner, copy2_fixed
from pip._internal.utils.glibc import libc_ver
from pip._internal.utils.misc import (
ask_path_exists,
@ -45,7 +38,6 @@ from pip._internal.utils.misc import (
build_url_from_netloc,
consume,
display_path,
ensure_dir,
format_size,
get_installed_version,
hide_url,
@ -263,63 +255,6 @@ class LocalFSAdapter(BaseAdapter):
pass
@contextmanager
def suppressed_cache_errors():
"""If we can't access the cache then we can just skip caching and process
requests as if caching wasn't enabled.
"""
try:
yield
except (OSError, IOError):
pass
class SafeFileCache(BaseCache):
"""
A file based cache which is safe to use even when the target directory may
not be accessible or writable.
"""
def __init__(self, directory):
# type: (str) -> None
assert directory is not None, "Cache directory must not be None."
super(SafeFileCache, self).__init__()
self.directory = directory
def _get_cache_path(self, name):
# type: (str) -> str
# From cachecontrol.caches.file_cache.FileCache._fn, brought into our
# class for backwards-compatibility and to avoid using a non-public
# method.
hashed = FileCache.encode(name)
parts = list(hashed[:5]) + [hashed]
return os.path.join(self.directory, *parts)
def get(self, key):
# type: (str) -> Optional[bytes]
path = self._get_cache_path(key)
with suppressed_cache_errors():
with open(path, 'rb') as f:
return f.read()
def set(self, key, value):
# type: (str, bytes) -> None
path = self._get_cache_path(key)
with suppressed_cache_errors():
ensure_dir(os.path.dirname(path))
with adjacent_tmp_file(path) as f:
f.write(value)
replace(f.name, path)
def delete(self, key):
# type: (str) -> None
path = self._get_cache_path(key)
with suppressed_cache_errors():
os.remove(path)
class InsecureHTTPAdapter(HTTPAdapter):
def cert_verify(self, conn, url, verify, cert):

71
src/pip/_internal/network/cache.py Normal file
View File

@ -0,0 +1,71 @@
"""HTTP cache implementation.
"""
import os
from contextlib import contextmanager
from pip._vendor.cachecontrol.cache import BaseCache
from pip._vendor.cachecontrol.caches import FileCache
from pip._internal.utils.filesystem import adjacent_tmp_file, replace
from pip._internal.utils.misc import ensure_dir
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
from typing import Optional
@contextmanager
def suppressed_cache_errors():
"""If we can't access the cache then we can just skip caching and process
requests as if caching wasn't enabled.
"""
try:
yield
except (OSError, IOError):
pass
class SafeFileCache(BaseCache):
"""
A file based cache which is safe to use even when the target directory may
not be accessible or writable.
"""
def __init__(self, directory):
# type: (str) -> None
assert directory is not None, "Cache directory must not be None."
super(SafeFileCache, self).__init__()
self.directory = directory
def _get_cache_path(self, name):
# type: (str) -> str
# From cachecontrol.caches.file_cache.FileCache._fn, brought into our
# class for backwards-compatibility and to avoid using a non-public
# method.
hashed = FileCache.encode(name)
parts = list(hashed[:5]) + [hashed]
return os.path.join(self.directory, *parts)
def get(self, key):
# type: (str) -> Optional[bytes]
path = self._get_cache_path(key)
with suppressed_cache_errors():
with open(path, 'rb') as f:
return f.read()
def set(self, key, value):
# type: (str, bytes) -> None
path = self._get_cache_path(key)
with suppressed_cache_errors():
ensure_dir(os.path.dirname(path))
with adjacent_tmp_file(path) as f:
f.write(value)
replace(f.name, path)
def delete(self, key):
# type: (str) -> None
path = self._get_cache_path(key)
with suppressed_cache_errors():
os.remove(path)

57
tests/unit/test_download.py
View File

@ -9,13 +9,11 @@ from tempfile import mkdtemp
import pytest
from mock import Mock, patch
from pip._vendor.cachecontrol.caches import FileCache
import pip
from pip._internal.download import (
CI_ENVIRONMENT_VARIABLES,
PipSession,
SafeFileCache,
_copy_source_tree,
_download_http_url,
parse_content_disposition,
@ -36,13 +34,6 @@ from tests.lib.filesystem import (
from tests.lib.path import Path
@pytest.fixture(scope="function")
def cache_tmpdir(tmpdir):
cache_dir = tmpdir.joinpath("cache")
cache_dir.mkdir(parents=True)
yield cache_dir
def test_unpack_http_url_with_urllib_response_without_content_type(data):
"""
It should download and unpack files even if no Content-Type header exists
@ -509,54 +500,6 @@ def test_unpack_file_url_excludes_expected_dirs(tmpdir, exclude_dir):
assert os.path.isdir(dst_included_dir)
class TestSafeFileCache:
"""
The no_perms test are useless on Windows since SafeFileCache uses
pip._internal.utils.filesystem.check_path_owner which is based on
os.geteuid which is absent on Windows.
"""
def test_cache_roundtrip(self, cache_tmpdir):
cache = SafeFileCache(cache_tmpdir)
assert cache.get("test key") is None
cache.set("test key", b"a test string")
assert cache.get("test key") == b"a test string"
cache.delete("test key")
assert cache.get("test key") is None
@pytest.mark.skipif("sys.platform == 'win32'")
def test_safe_get_no_perms(self, cache_tmpdir, monkeypatch):
os.chmod(cache_tmpdir, 000)
monkeypatch.setattr(os.path, "exists", lambda x: True)
cache = SafeFileCache(cache_tmpdir)
cache.get("foo")
@pytest.mark.skipif("sys.platform == 'win32'")
def test_safe_set_no_perms(self, cache_tmpdir):
os.chmod(cache_tmpdir, 000)
cache = SafeFileCache(cache_tmpdir)
cache.set("foo", b"bar")
@pytest.mark.skipif("sys.platform == 'win32'")
def test_safe_delete_no_perms(self, cache_tmpdir):
os.chmod(cache_tmpdir, 000)
cache = SafeFileCache(cache_tmpdir)
cache.delete("foo")
def test_cache_hashes_are_same(self, cache_tmpdir):
cache = SafeFileCache(cache_tmpdir)
key = "test key"
fake_cache = Mock(
FileCache, directory=cache.directory, encode=FileCache.encode
)
assert cache._get_cache_path(key) == FileCache._fn(fake_cache, key)
class TestPipSession:
def test_cache_defaults_off(self):

62
tests/unit/test_networking_cache.py Normal file
View File

@ -0,0 +1,62 @@
import os
import pytest
from mock import Mock
from pip._vendor.cachecontrol.caches import FileCache
from pip._internal.network.cache import SafeFileCache
@pytest.fixture(scope="function")
def cache_tmpdir(tmpdir):
cache_dir = tmpdir.joinpath("cache")
cache_dir.mkdir(parents=True)
yield cache_dir
class TestSafeFileCache:
"""
The no_perms test are useless on Windows since SafeFileCache uses
pip._internal.utils.filesystem.check_path_owner which is based on
os.geteuid which is absent on Windows.
"""
def test_cache_roundtrip(self, cache_tmpdir):
cache = SafeFileCache(cache_tmpdir)
assert cache.get("test key") is None
cache.set("test key", b"a test string")
assert cache.get("test key") == b"a test string"
cache.delete("test key")
assert cache.get("test key") is None
@pytest.mark.skipif("sys.platform == 'win32'")
def test_safe_get_no_perms(self, cache_tmpdir, monkeypatch):
os.chmod(cache_tmpdir, 000)
monkeypatch.setattr(os.path, "exists", lambda x: True)
cache = SafeFileCache(cache_tmpdir)
cache.get("foo")
@pytest.mark.skipif("sys.platform == 'win32'")
def test_safe_set_no_perms(self, cache_tmpdir):
os.chmod(cache_tmpdir, 000)
cache = SafeFileCache(cache_tmpdir)
cache.set("foo", b"bar")
@pytest.mark.skipif("sys.platform == 'win32'")
def test_safe_delete_no_perms(self, cache_tmpdir):
os.chmod(cache_tmpdir, 000)
cache = SafeFileCache(cache_tmpdir)
cache.delete("foo")
def test_cache_hashes_are_same(self, cache_tmpdir):
cache = SafeFileCache(cache_tmpdir)
key = "test key"
fake_cache = Mock(
FileCache, directory=cache.directory, encode=FileCache.encode
)
assert cache._get_cache_path(key) == FileCache._fn(fake_cache, key)